diff --git a/src/core/grid_based_algorithms/lb_interface.cpp b/src/core/grid_based_algorithms/lb_interface.cpp index 9e3e371a957..ed1c1f40940 100644 --- a/src/core/grid_based_algorithms/lb_interface.cpp +++ b/src/core/grid_based_algorithms/lb_interface.cpp @@ -531,11 +531,12 @@ lb_lbnode_get_pressure_tensor(const Utils::Vector3i &ind) { ::Communication::Result::one_rank, Walberla::get_node_pressure_tensor, ind); - // reverts the correction done by walberla - // auto const revert_factor = - // lb_lbfluid_get_viscosity() / (lb_lbfluid_get_viscosity() + 1.0 - // / 6.0); stress[1] /= revert_factor; stress[3] /= revert_factor; - // stress[4] /= revert_factor; + // correction factor + auto const adjust_factor = + lb_lbfluid_get_viscosity() / (lb_lbfluid_get_viscosity() + 1.0 / 6.0); + stress[1] *= adjust_factor; + stress[3] *= adjust_factor; + stress[4] *= adjust_factor; return stress; } diff --git a/src/core/grid_based_algorithms/lb_interface.hpp b/src/core/grid_based_algorithms/lb_interface.hpp index d6e8dc9836f..119386d6466 100644 --- a/src/core/grid_based_algorithms/lb_interface.hpp +++ b/src/core/grid_based_algorithms/lb_interface.hpp @@ -247,3 +247,4 @@ void lb_lbfluid_add_force_at_pos(const Utils::Vector3d &pos, double lb_lbfluid_get_interpolated_density(const Utils::Vector3d &pos); #endif + diff --git a/src/core/grid_based_algorithms/lb_walberla_instance.cpp b/src/core/grid_based_algorithms/lb_walberla_instance.cpp index 8f554023a27..00fe4cff671 100644 --- a/src/core/grid_based_algorithms/lb_walberla_instance.cpp +++ b/src/core/grid_based_algorithms/lb_walberla_instance.cpp @@ -110,3 +110,4 @@ void mpi_destruct_lb_walberla() { Communication::mpiCallbacks().call_all(destruct_lb_walberla); } #endif + diff --git a/src/core/grid_based_algorithms/lb_walberla_instance.hpp b/src/core/grid_based_algorithms/lb_walberla_instance.hpp index e6d0247a329..5cc61f51892 100644 --- a/src/core/grid_based_algorithms/lb_walberla_instance.hpp +++ b/src/core/grid_based_algorithms/lb_walberla_instance.hpp @@ -64,3 +64,4 @@ void mpi_destruct_lb_walberla(); #endif // LB_WALBERLA #endif + diff --git a/src/python/espressomd/lb.pxd b/src/python/espressomd/lb.pxd index cccc1304635..a28d5b4e12d 100644 --- a/src/python/espressomd/lb.pxd +++ b/src/python/espressomd/lb.pxd @@ -210,3 +210,4 @@ cdef inline python_lbfluid_set_gamma(p_gamma) except +: c_gamma = p_gamma # call c-function lb_lbcoupling_set_gamma(c_gamma) + diff --git a/src/python/espressomd/lb.pyx b/src/python/espressomd/lb.pyx index ce3044b878c..4a07c530e95 100644 --- a/src/python/espressomd/lb.pyx +++ b/src/python/espressomd/lb.pyx @@ -396,7 +396,6 @@ IF LB_WALBERLA: """ Initialize the lattice-Boltzmann method for hydrodynamic flow using waLBerla. See :class:`HydrodynamicInteraction` for the list of parameters. - """ def _set_params_in_es_core(self): @@ -615,7 +614,6 @@ def _add_lb_slice_properties(): """ Setter function that sets attribute on every member of lb_slice. If values contains only one element, all members are set to it. - """ indices = [lb_slice.x_indices, lb_slice.y_indices, lb_slice.z_indices] @@ -669,3 +667,4 @@ def _add_lb_slice_properties(): _add_lb_slice_properties() + diff --git a/src/walberla_bridge/LBWalberlaBase.hpp b/src/walberla_bridge/LBWalberlaBase.hpp index 8a7bdb1595e..8d79463ce39 100644 --- a/src/walberla_bridge/LBWalberlaBase.hpp +++ b/src/walberla_bridge/LBWalberlaBase.hpp @@ -179,3 +179,4 @@ enum class OutputVTK : unsigned { }; #endif // LB_WALBERLA_H + diff --git a/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp b/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp index a3a74ab031e..61354c681ff 100644 --- a/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp +++ b/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp @@ -75,3 +75,4 @@ class LBWalberlaD3Q19FluctuatingMRT : public LBWalberlaImpl { } // namespace walberla #undef LatticeModelName + diff --git a/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp b/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp index 8455da4ff84..312209009b8 100644 --- a/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp +++ b/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp @@ -49,3 +49,4 @@ class LBWalberlaD3Q19MRT : public LBWalberlaImpl { } // namespace walberla #undef LatticeModelName + diff --git a/src/walberla_bridge/generated_kernels/Dynamic_UBB.cpp b/src/walberla_bridge/generated_kernels/Dynamic_UBB.cpp index 955dc005041..64e652aff6d 100644 --- a/src/walberla_bridge/generated_kernels/Dynamic_UBB.cpp +++ b/src/walberla_bridge/generated_kernels/Dynamic_UBB.cpp @@ -19,9 +19,11 @@ #include -#include "Dynamic_UBB.h" #include "core/DataTypes.h" #include "core/Macros.h" +#include "Dynamic_UBB.h" + + #define FUNC_PREFIX @@ -43,102 +45,38 @@ namespace lbm { #endif namespace internal_dynamic_ubb_boundary_Dynamic_UBB { -static FUNC_PREFIX void dynamic_ubb_boundary_Dynamic_UBB( - uint8_t *RESTRICT const _data_indexVector, double *RESTRICT _data_pdfs, - int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, - int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, - int64_t indexVectorSize) { - - const int32_t f_in_inv_dir_idx[] = {0, 2, 1, 4, 3, 6, 5, 10, 9, 8, - 7, 16, 15, 18, 17, 12, 11, 14, 13}; - const int32_t f_out_offsets_x[] = {0, 0, 0, -1, 1, 0, 0, -1, 1, -1, - 1, 0, 0, -1, 1, 0, 0, -1, 1}; - const int32_t f_out_offsets_y[] = {0, 1, -1, 0, 0, 0, 0, 1, 1, -1, - -1, 1, -1, 0, 0, 1, -1, 0, 0}; - const int32_t f_out_offsets_z[] = {0, 0, 0, 0, 0, 1, -1, 0, 0, 0, - 0, 1, 1, 1, 1, -1, -1, -1, -1}; - - const double weights[] = { - 0.333333333333333, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778}; - - const int32_t neighbour_offset_x[] = {0, 0, 0, -1, 1, 0, 0, -1, 1, -1, - 1, 0, 0, -1, 1, 0, 0, -1, 1}; - const int32_t neighbour_offset_y[] = {0, 1, -1, 0, 0, 0, 0, 1, 1, -1, - -1, 1, -1, 0, 0, 1, -1, 0, 0}; - const int32_t neighbour_offset_z[] = {0, 0, 0, 0, 0, 1, -1, 0, 0, 0, - 0, 1, 1, 1, 1, -1, -1, -1, -1}; - - for (int64_t ctr_0 = 0; ctr_0 < indexVectorSize; ctr_0 += 1) { - const int32_t x = *((int32_t *)(&_data_indexVector[40 * ctr_0])); - const int32_t y = *((int32_t *)(&_data_indexVector[40 * ctr_0 + 4])); - const int32_t z = *((int32_t *)(&_data_indexVector[40 * ctr_0 + 8])); - const int32_t dir = *((int32_t *)(&_data_indexVector[40 * ctr_0 + 12])); - const double vel0Term = - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_1 + _stride_pdfs_2 * z + 8 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z + 4 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z + _stride_pdfs_2 + 14 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z - _stride_pdfs_2 + 18 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 + _stride_pdfs_1 * y - - _stride_pdfs_1 + _stride_pdfs_2 * z + 10 * _stride_pdfs_3]; - const double vel1Term = - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + _stride_pdfs_1 + - _stride_pdfs_2 * z + _stride_pdfs_2 + 11 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + _stride_pdfs_1 + - _stride_pdfs_2 * z + _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + _stride_pdfs_1 + - _stride_pdfs_2 * z - _stride_pdfs_2 + 15 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x - _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_1 + _stride_pdfs_2 * z + 7 * _stride_pdfs_3]; - const double vel2Term = - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + - _stride_pdfs_2 * z + _stride_pdfs_2 + 5 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y - _stride_pdfs_1 + - _stride_pdfs_2 * z + _stride_pdfs_2 + 12 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x - _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z + _stride_pdfs_2 + 13 * _stride_pdfs_3]; - const double rho = - vel0Term + vel1Term + vel2Term + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + - _stride_pdfs_2 * z - _stride_pdfs_2 + 6 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + - _stride_pdfs_2 * z] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y - _stride_pdfs_1 + - _stride_pdfs_2 * z + 2 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y - _stride_pdfs_1 + - _stride_pdfs_2 * z - _stride_pdfs_2 + 16 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x - _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z + 3 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x - _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z - _stride_pdfs_2 + 17 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x - _stride_pdfs_0 + _stride_pdfs_1 * y - - _stride_pdfs_1 + _stride_pdfs_2 * z + 9 * _stride_pdfs_3]; - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + _stride_pdfs_2 * z + - _stride_pdfs_3 * f_in_inv_dir_idx[dir]] = - -rho * - (6.0 * *((double *)(&_data_indexVector[40 * ctr_0 + 16])) * - neighbour_offset_x[dir] + - 6.0 * *((double *)(&_data_indexVector[40 * ctr_0 + 24])) * - neighbour_offset_y[dir] + - 6.0 * *((double *)(&_data_indexVector[40 * ctr_0 + 32])) * - neighbour_offset_z[dir]) * - weights[dir] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 * f_out_offsets_x[dir] + - _stride_pdfs_1 * y + _stride_pdfs_1 * f_out_offsets_y[dir] + - _stride_pdfs_2 * z + _stride_pdfs_2 * f_out_offsets_z[dir] + - _stride_pdfs_3 * dir]; - } +static FUNC_PREFIX void dynamic_ubb_boundary_Dynamic_UBB(uint8_t * RESTRICT const _data_indexVector, double * RESTRICT _data_pdfs, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t indexVectorSize) +{ + + const int32_t f_in_inv_dir_idx [] = { 0,2,1,4,3,6,5,10,9,8,7,16,15,18,17,12,11,14,13 }; + const int32_t f_out_offsets_x [] = { 0,0,0,-1,1,0,0,-1,1,-1,1,0,0,-1,1,0,0,-1,1 }; + const int32_t f_out_offsets_y [] = { 0,1,-1,0,0,0,0,1,1,-1,-1,1,-1,0,0,1,-1,0,0 }; + const int32_t f_out_offsets_z [] = { 0,0,0,0,0,1,-1,0,0,0,0,1,1,1,1,-1,-1,-1,-1 }; + + + const double weights [] = { 0.333333333333333,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778 }; + + + + const int32_t neighbour_offset_x [] = { 0,0,0,-1,1,0,0,-1,1,-1,1,0,0,-1,1,0,0,-1,1 }; + const int32_t neighbour_offset_y [] = { 0,1,-1,0,0,0,0,1,1,-1,-1,1,-1,0,0,1,-1,0,0 }; + const int32_t neighbour_offset_z [] = { 0,0,0,0,0,1,-1,0,0,0,0,1,1,1,1,-1,-1,-1,-1 }; + + for (int64_t ctr_0 = 0; ctr_0 < indexVectorSize; ctr_0 += 1) + { + const int32_t x = *((int32_t *)(& _data_indexVector[40*ctr_0])); + const int32_t y = *((int32_t *)(& _data_indexVector[40*ctr_0 + 4])); + const int32_t z = *((int32_t *)(& _data_indexVector[40*ctr_0 + 8])); + const int32_t dir = *((int32_t *)(& _data_indexVector[40*ctr_0 + 12])); + const double vel0Term = _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_1 + _stride_pdfs_2*z + 8*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z + 4*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z + _stride_pdfs_2 + 14*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z - _stride_pdfs_2 + 18*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0 + _stride_pdfs_1*y - _stride_pdfs_1 + _stride_pdfs_2*z + 10*_stride_pdfs_3]; + const double vel1Term = _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_1 + _stride_pdfs_2*z + _stride_pdfs_2 + 11*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_1 + _stride_pdfs_2*z + _stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_1 + _stride_pdfs_2*z - _stride_pdfs_2 + 15*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x - _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_1 + _stride_pdfs_2*z + 7*_stride_pdfs_3]; + const double vel2Term = _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_2*z + _stride_pdfs_2 + 5*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y - _stride_pdfs_1 + _stride_pdfs_2*z + _stride_pdfs_2 + 12*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x - _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z + _stride_pdfs_2 + 13*_stride_pdfs_3]; + const double rho = vel0Term + vel1Term + vel2Term + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_2*z - _stride_pdfs_2 + 6*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_2*z] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y - _stride_pdfs_1 + _stride_pdfs_2*z + 2*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y - _stride_pdfs_1 + _stride_pdfs_2*z - _stride_pdfs_2 + 16*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x - _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z + 3*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x - _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z - _stride_pdfs_2 + 17*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x - _stride_pdfs_0 + _stride_pdfs_1*y - _stride_pdfs_1 + _stride_pdfs_2*z + 9*_stride_pdfs_3]; + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_2*z + _stride_pdfs_3*f_in_inv_dir_idx[dir]] = -rho*(6.0**((double *)(& _data_indexVector[40*ctr_0 + 16]))*neighbour_offset_x[dir] + 6.0**((double *)(& _data_indexVector[40*ctr_0 + 24]))*neighbour_offset_y[dir] + 6.0**((double *)(& _data_indexVector[40*ctr_0 + 32]))*neighbour_offset_z[dir])*weights[dir] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0*f_out_offsets_x[dir] + _stride_pdfs_1*y + _stride_pdfs_1*f_out_offsets_y[dir] + _stride_pdfs_2*z + _stride_pdfs_2*f_out_offsets_z[dir] + _stride_pdfs_3*dir]; + } +} } -} // namespace internal_dynamic_ubb_boundary_Dynamic_UBB + #ifdef __GNUC__ #pragma GCC diagnostic pop @@ -148,34 +86,47 @@ static FUNC_PREFIX void dynamic_ubb_boundary_Dynamic_UBB( #pragma pop #endif -void Dynamic_UBB::run_impl(IBlock *block, IndexVectors::Type type) { - auto *indexVectors = block->getData(indexVectorID); - int64_t indexVectorSize = int64_c(indexVectors->indexVector(type).size()); - if (indexVectorSize == 0) - return; - auto pointer = indexVectors->pointerCpu(type); +void Dynamic_UBB::run_impl(IBlock * block, IndexVectors::Type type) +{ + auto * indexVectors = block->getData(indexVectorID); + int64_t indexVectorSize = int64_c( indexVectors->indexVector(type).size() ); + if( indexVectorSize == 0) + return; - uint8_t *_data_indexVector = reinterpret_cast(pointer); + + auto pointer = indexVectors->pointerCpu(type); + - auto pdfs = block->getData>(pdfsID); + uint8_t * _data_indexVector = reinterpret_cast(pointer); - WALBERLA_ASSERT_GREATER_EQUAL(0, -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = pdfs->dataAt(0, 0, 0, 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_dynamic_ubb_boundary_Dynamic_UBB::dynamic_ubb_boundary_Dynamic_UBB( - _data_indexVector, _data_pdfs, _stride_pdfs_0, _stride_pdfs_1, - _stride_pdfs_2, _stride_pdfs_3, indexVectorSize); + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + + + WALBERLA_ASSERT_GREATER_EQUAL(0, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(0, 0, 0, 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_dynamic_ubb_boundary_Dynamic_UBB::dynamic_ubb_boundary_Dynamic_UBB(_data_indexVector, _data_pdfs, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, indexVectorSize); } -void Dynamic_UBB::run(IBlock *block) { run_impl(block, IndexVectors::ALL); } +void Dynamic_UBB::run(IBlock * block) +{ + run_impl(block, IndexVectors::ALL); +} -void Dynamic_UBB::inner(IBlock *block) { run_impl(block, IndexVectors::INNER); } +void Dynamic_UBB::inner(IBlock * block) +{ + run_impl(block, IndexVectors::INNER); +} -void Dynamic_UBB::outer(IBlock *block) { run_impl(block, IndexVectors::OUTER); } +void Dynamic_UBB::outer(IBlock * block) +{ + run_impl(block, IndexVectors::OUTER); +} } // namespace lbm } // namespace walberla + diff --git a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.cpp b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.cpp index 2529f01fa99..49d77b5a204 100644 --- a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.cpp +++ b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.cpp @@ -18,1783 +18,1209 @@ #include -#include "FluctuatingMRTLatticeModel.h" #include "core/DataTypes.h" #include "core/Macros.h" #include "lbm/field/PdfField.h" #include "lbm/sweeps/Streaming.h" +#include "FluctuatingMRTLatticeModel.h" #ifdef _MSC_VER -#pragma warning(disable : 4458) +# pragma warning( disable : 4458 ) #endif #define FUNC_PREFIX -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#pragma GCC diagnostic ignored "-Wshadow" -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-parameter" +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wfloat-equal" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Wunused-parameter" #endif + #include "philox_rand.h" + + using namespace std; namespace walberla { namespace lbm { namespace internal_kernel_streamCollide { -static FUNC_PREFIX void kernel_streamCollide( - double *RESTRICT const _data_force, double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, - int64_t const _size_force_1, int64_t const _size_force_2, - int64_t const _stride_force_0, int64_t const _stride_force_1, - int64_t const _stride_force_2, int64_t const _stride_force_3, - int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, - int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, - int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, - int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, - uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, - double kT, double omega_bulk, double omega_even, double omega_odd, - double omega_shear, uint32_t seed, uint32_t time_step) { - const double xi_25 = -omega_bulk; - const double xi_36 = -omega_shear; - const double xi_37 = xi_36 + 2.0; - const double xi_38 = xi_37 * 0.5; - const double xi_43 = xi_37 * 0.0833333333333333; - const double xi_48 = xi_37 * 0.166666666666667; - const double xi_58 = xi_37 * 0.25; - const double xi_63 = xi_37 * 0.0416666666666667; - const double xi_90 = 2.4494897427831779; - const double xi_115 = omega_odd * 0.25; - const double xi_131 = omega_odd * 0.0833333333333333; - const double xi_196 = omega_shear * 0.25; - const double xi_211 = omega_odd * 0.0416666666666667; - const double xi_213 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_120 = rr_0 * 0.166666666666667; - const double xi_186 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - for (int64_t ctr_0 = 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) { - - double random_7_0; - double random_7_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 7, seed, - random_7_0, random_7_1); - - double random_6_0; - double random_6_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 6, seed, - random_6_0, random_6_1); - - double random_5_0; - double random_5_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 5, seed, - random_5_0, random_5_1); - - double random_4_0; - double random_4_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 4, seed, - random_4_0, random_4_1); - - double random_3_0; - double random_3_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, - random_3_0, random_3_1); - - double random_2_0; - double random_2_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, - random_2_0, random_2_1); - - double random_1_0; - double random_1_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, - random_1_0, random_1_1); - - double random_0_0; - double random_0_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, - random_0_0, random_0_1); - - const double xi_0 = - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_1 = - xi_0 + _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_2 = _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_3 = _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_4 = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_5 = _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_6 = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_8 = - -_data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_9 = - xi_8 - - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_10 = - -_data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_11 = - -_data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_12 = - -_data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -_data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - const double xi_15 = - -_data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -_data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_18 = -_data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = - -_data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -_data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_23 = -_data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_24 = xi_17 + xi_22 + xi_23 + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_42 = - 0.166666666666667 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_50 = - 0.166666666666667 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_54 = - 0.166666666666667 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_57 = - 0.5 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_61 = - 0.0833333333333333 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_65 = - 0.0833333333333333 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_75 = - 0.0833333333333333 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_93 = -_data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_94 = xi_93 + - 3.0 * _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_95 = - omega_even * - (xi_94 - 3.0 * _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_96 = - 2.0 * _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_97 = - xi_96 + - 5.0 * _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 5.0 * _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_98 = - omega_even * - (xi_94 + xi_97 - - 2.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] - - 2.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0] - - 5.0 * - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 5.0 * - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 5.0 * _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + - _stride_pdfs_0] - - 5.0 * _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - - _stride_pdfs_0]); - const double xi_101 = -_data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_102 = xi_101 + xi_18; - const double xi_103 = - -_data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_106 = - -_data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_107 = xi_106 + xi_11 + xi_15 + xi_21; - const double xi_109 = - 2.0 * - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_110 = - 2.0 * - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_111 = - 2.0 * - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 2.0 * _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_112 = - omega_even * - (xi_109 + xi_110 + xi_111 + xi_93 + xi_97 - - 4.0 * _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] - - 4.0 * _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0] - - 7.0 * - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 5.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - 5.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_113 = - xi_101 + _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_114 = xi_113 + xi_14 + xi_22 + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_116 = xi_114 * xi_115; - const double xi_118 = - xi_103 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_122 = random_5_1 - 0.5; - const double xi_127 = - 2.0 * _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_128 = - 2.0 * _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_129 = - -2.0 * - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - 2.0 * _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_130 = -xi_127 + xi_128 + xi_129 + xi_14 + xi_19 + xi_2; - const double xi_132 = xi_130 * xi_131; - const double xi_133 = random_3_0 - 0.5; - const double xi_138 = random_0_1 - 0.5; - const double xi_142 = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_156 = - xi_106 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_157 = - xi_12 + xi_156 + xi_20 + - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_158 = xi_115 * xi_157; - const double xi_159 = random_4_1 - 0.5; - const double xi_161 = xi_1 + xi_127 - xi_128 + xi_129 + xi_13; - const double xi_162 = xi_131 * xi_161; - const double xi_163 = random_4_0 - 0.5; - const double xi_168 = _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_169 = xi_102 + xi_168 + xi_23 + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_170 = xi_115 * xi_169; - const double xi_173 = random_5_0 - 0.5; - const double xi_175 = -xi_109 - xi_110 + xi_111 + xi_24 + xi_3; - const double xi_176 = xi_131 * xi_175; - const double xi_177 = random_3_1 - 0.5; - const double xi_184 = xi_112 * 0.0138888888888889; - const double xi_205 = xi_98 * -0.00714285714285714; - const double xi_207 = xi_95 * 0.025; - const double xi_212 = xi_175 * xi_211; - const double xi_214 = xi_169 * xi_213; - const double xi_223 = xi_130 * xi_211; - const double xi_224 = xi_114 * xi_213; - const double xi_232 = xi_98 * 0.0178571428571429; - const double xi_238 = xi_157 * xi_213; - const double xi_239 = xi_161 * xi_211; - const double vel0Term = - xi_1 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double vel1Term = - xi_2 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double vel2Term = - xi_3 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double rho = vel0Term + vel1Term + vel2Term + xi_4 + xi_5 + xi_6 + - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_7 = 1 / (rho); - const double xi_86 = kT * rho; - const double xi_87 = - sqrt(xi_86 * (-((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0)); - const double xi_88 = xi_87 * (random_6_0 - 0.5) * 3.7416573867739413; - const double xi_89 = xi_87 * (random_7_0 - 0.5) * 5.4772255750516612; - const double xi_91 = - xi_90 * sqrt(xi_86 * (-((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0)) * - (random_2_1 - 0.5); - const double xi_92 = xi_87 * (random_6_1 - 0.5) * 8.3666002653407556; - const double xi_123 = - sqrt(xi_86 * (-((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0)); - const double xi_124 = xi_123 * 1.4142135623730951; - const double xi_125 = xi_124 * 0.5; - const double xi_126 = xi_122 * xi_125; - const double xi_134 = xi_123 * xi_90; - const double xi_135 = xi_134 * 0.166666666666667; - const double xi_136 = xi_133 * xi_135; - const double xi_137 = -xi_132 - xi_136; - const double xi_139 = - sqrt(xi_86 * (-((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0)); - const double xi_140 = xi_139 * 0.5; - const double xi_141 = xi_138 * xi_140; - const double xi_146 = - xi_112 * -0.0198412698412698 + xi_88 * -0.119047619047619; - const double xi_148 = xi_139 * (random_0_0 - 0.5) * 1.7320508075688772; - const double xi_152 = xi_132 + xi_136; - const double xi_160 = xi_125 * xi_159; - const double xi_164 = xi_135 * xi_163; - const double xi_165 = xi_162 + xi_164; - const double xi_167 = -xi_162 - xi_164; - const double xi_174 = xi_125 * xi_173; - const double xi_178 = xi_135 * xi_177; - const double xi_179 = -xi_176 - xi_178; - const double xi_181 = xi_176 + xi_178; - const double xi_182 = xi_138 * xi_139 * 0.25; - const double xi_185 = xi_88 * 0.0833333333333333; - const double xi_195 = xi_140 * (random_1_0 - 0.5); - const double xi_204 = xi_140 * (random_2_0 - 0.5); - const double xi_208 = xi_92 * -0.0142857142857143; - const double xi_209 = xi_89 * 0.05; - const double xi_215 = xi_134 * 0.0833333333333333; - const double xi_216 = xi_177 * xi_215; - const double xi_217 = xi_124 * 0.25; - const double xi_218 = xi_173 * xi_217; - const double xi_220 = - xi_112 * -0.00396825396825397 + xi_88 * -0.0238095238095238; - const double xi_225 = xi_133 * xi_215; - const double xi_226 = xi_122 * xi_217; - const double xi_230 = -xi_182; - const double xi_233 = xi_92 * 0.0357142857142857; - const double xi_235 = xi_140 * (random_1_1 - 0.5); - const double xi_240 = xi_159 * xi_217; - const double xi_241 = xi_163 * xi_215; - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_26 = - u_0 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_27 = xi_26 * 0.333333333333333; - const double xi_33 = -xi_27; - const double xi_99 = rho * (u_0 * u_0); - const double xi_153 = rho * u_0; - const double xi_154 = - -vel0Term + xi_142 + xi_153 + xi_4 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_155 = xi_120 * xi_154; - const double xi_191 = xi_154 * xi_186; - const double u_1 = - xi_7 * - (vel1Term + xi_16 + xi_19 + xi_8 + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_28 = - u_1 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_29 = xi_28 * 0.333333333333333; - const double xi_34 = -xi_29; - const double xi_56 = u_1 * 0.5; - const double xi_59 = - xi_58 * (u_0 * xi_57 + - xi_56 * _data_force_20_30_10[_stride_force_0 * ctr_0]); - const double xi_60 = -xi_59; - const double xi_104 = rho * (u_1 * u_1); - const double xi_105 = xi_103 + xi_104 + xi_9; - const double xi_117 = rho * u_1; - const double xi_119 = - -vel1Term + xi_117 + xi_118 + xi_5 + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_121 = xi_119 * xi_120; - const double xi_187 = xi_119 * xi_186; - const double xi_197 = - xi_196 * - (u_0 * xi_117 + xi_118 + xi_8 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]); - const double xi_198 = -xi_195 - xi_197; - const double xi_199 = xi_195 + xi_197; - const double u_2 = - xi_7 * - (vel2Term + xi_21 + xi_24 + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_30 = - u_2 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_31 = xi_30 * 0.333333333333333; - const double xi_32 = (xi_25 + 2.0) * (xi_27 + xi_29 + xi_31); - const double xi_35 = xi_30 * 0.666666666666667 + xi_33 + xi_34; - const double xi_39 = -xi_31; - const double xi_40 = xi_28 * 0.666666666666667 + xi_33 + xi_39; - const double xi_41 = xi_26 * 0.666666666666667 + xi_34 + xi_39; - const double xi_44 = xi_35 * xi_43; - const double xi_45 = -xi_44; - const double xi_46 = xi_41 * xi_43; - const double xi_47 = -xi_46; - const double xi_49 = xi_40 * xi_48 + xi_45 + xi_47; - const double xi_51 = xi_40 * xi_43; - const double xi_52 = -xi_51; - const double xi_53 = xi_41 * xi_48 + xi_45 + xi_52; - const double xi_55 = xi_35 * xi_48 + xi_47 + xi_52; - const double xi_62 = xi_46 - xi_61; - const double xi_64 = -xi_35 * xi_63; - const double xi_66 = xi_32 * 0.125; - const double xi_67 = xi_51 + xi_66; - const double xi_68 = xi_65 + xi_67; - const double xi_69 = xi_64 + xi_68; - const double xi_70 = xi_46 + xi_61; - const double xi_71 = -xi_65 + xi_67; - const double xi_72 = xi_64 + xi_71; - const double xi_73 = - xi_58 * (u_2 * xi_57 + - xi_56 * _data_force_20_32_10[_stride_force_0 * ctr_0]); - const double xi_74 = -xi_41 * xi_63; - const double xi_76 = xi_44 + xi_75; - const double xi_77 = xi_74 + xi_76; - const double xi_78 = -xi_73; - const double xi_79 = - xi_58 * (u_0 * 0.5 * _data_force_20_32_10[_stride_force_0 * ctr_0] + - u_2 * 0.5 * _data_force_20_30_10[_stride_force_0 * ctr_0]); - const double xi_80 = -xi_79; - const double xi_81 = -xi_40 * xi_63; - const double xi_82 = xi_66 + xi_76 + xi_81; - const double xi_83 = xi_44 - xi_75; - const double xi_84 = xi_74 + xi_83; - const double xi_85 = xi_66 + xi_81 + xi_83; - const double xi_100 = rho * (u_2 * u_2); - const double xi_108 = - omega_bulk * (xi_100 + xi_102 + xi_105 + xi_107 + xi_17 + xi_22 + - xi_99 + _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]); - const double xi_143 = -xi_100 + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_144 = - omega_shear * (xi_0 + xi_105 + xi_142 + xi_143 + xi_16 - - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]); - const double xi_145 = xi_144 * 0.125; - const double xi_147 = - omega_shear * - (xi_103 - xi_104 + xi_107 + xi_143 + xi_9 + xi_96 + xi_99 * 2.0 - - 2.0 * - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 2.0 * - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_149 = - xi_147 * -0.0416666666666667 + xi_148 * -0.166666666666667; - const double xi_150 = xi_149 + xi_89 * -0.1 + xi_95 * -0.05; - const double xi_151 = xi_141 + xi_145 + xi_146 + xi_150 + - xi_92 * 0.0285714285714286 + - xi_98 * 0.0142857142857143; - const double xi_166 = - xi_146 + xi_147 * 0.0833333333333333 + xi_148 * 0.333333333333333 + - xi_92 * -0.0714285714285714 + xi_98 * -0.0357142857142857; - const double xi_171 = - rho * u_2 - vel2Term + xi_101 + xi_106 + xi_168 + xi_6 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_172 = xi_120 * xi_171; - const double xi_180 = xi_112 * 0.0158730158730159 - xi_141 - xi_145 + - xi_150 + xi_88 * 0.0952380952380952 + - xi_92 * -0.0428571428571429 + - xi_98 * -0.0214285714285714; - const double xi_183 = xi_144 * 0.0625; - const double xi_188 = - xi_108 * 0.0416666666666667 + xi_91 * 0.0833333333333333; - const double xi_189 = xi_187 + xi_188; - const double xi_190 = - xi_152 + xi_182 + xi_183 + xi_184 + xi_185 + xi_189; - const double xi_192 = - xi_147 * 0.0208333333333333 + xi_148 * 0.0833333333333333; - const double xi_193 = -xi_191 + xi_192; - const double xi_194 = xi_167 + xi_193; - const double xi_200 = xi_191 + xi_192; - const double xi_201 = xi_165 + xi_200; - const double xi_202 = -xi_187 + xi_188; - const double xi_203 = - xi_137 + xi_182 + xi_183 + xi_184 + xi_185 + xi_202; - const double xi_206 = - xi_196 * (u_2 * xi_117 + xi_113 + xi_17 + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]); - const double xi_210 = - xi_149 + xi_204 + xi_205 + xi_206 + xi_207 + xi_208 + xi_209; - const double xi_219 = xi_171 * xi_186; - const double xi_221 = xi_219 + xi_220; - const double xi_222 = -xi_212 + xi_214 - xi_216 + xi_218 + xi_221; - const double xi_227 = xi_189 - xi_223 + xi_224 - xi_225 + xi_226; - const double xi_228 = xi_202 + xi_223 - xi_224 + xi_225 - xi_226; - const double xi_229 = - xi_149 - xi_204 + xi_205 - xi_206 + xi_207 + xi_208 + xi_209; - const double xi_231 = -xi_183; - const double xi_234 = - xi_181 + xi_188 + xi_221 + xi_230 + xi_231 + xi_232 + xi_233; - const double xi_236 = - xi_196 * - (u_2 * xi_153 + xi_10 + xi_156 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_237 = -xi_235 - xi_236; - const double xi_242 = xi_193 - xi_238 + xi_239 - xi_240 + xi_241; - const double xi_243 = xi_235 + xi_236; - const double xi_244 = xi_200 + xi_238 - xi_239 + xi_240 - xi_241; - const double xi_245 = -xi_219 + xi_220; - const double xi_246 = xi_212 - xi_214 + xi_216 - xi_218 + xi_245; - const double xi_247 = - xi_179 + xi_188 + xi_230 + xi_231 + xi_232 + xi_233 + xi_245; - const double forceTerm_0 = - xi_32 * -1.5 - xi_35 * xi_38 - xi_38 * xi_40 - xi_38 * xi_41; - const double forceTerm_1 = xi_42 + xi_49; - const double forceTerm_2 = -xi_42 + xi_49; - const double forceTerm_3 = -xi_50 + xi_53; - const double forceTerm_4 = xi_50 + xi_53; - const double forceTerm_5 = xi_54 + xi_55; - const double forceTerm_6 = -xi_54 + xi_55; - const double forceTerm_7 = xi_60 + xi_62 + xi_69; - const double forceTerm_8 = xi_59 + xi_69 + xi_70; - const double forceTerm_9 = xi_59 + xi_62 + xi_72; - const double forceTerm_10 = xi_60 + xi_70 + xi_72; - const double forceTerm_11 = xi_68 + xi_73 + xi_77; - const double forceTerm_12 = xi_71 + xi_77 + xi_78; - const double forceTerm_13 = xi_62 + xi_80 + xi_82; - const double forceTerm_14 = xi_70 + xi_79 + xi_82; - const double forceTerm_15 = xi_68 + xi_78 + xi_84; - const double forceTerm_16 = xi_71 + xi_73 + xi_84; - const double forceTerm_17 = xi_62 + xi_79 + xi_85; - const double forceTerm_18 = xi_70 + xi_80 + xi_85; - _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_0 + xi_108 * -0.5 + xi_112 * 0.0238095238095238 + - xi_88 * 0.142857142857143 + xi_89 * 0.2 - xi_91 + - xi_92 * 0.0857142857142857 + xi_95 * 0.1 + - xi_98 * 0.0428571428571429 + - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_1 - xi_116 + xi_121 - xi_126 + xi_137 + xi_151 + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_2 + xi_116 - xi_121 + xi_126 + xi_151 + xi_152 + - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_3 - xi_155 + xi_158 + xi_160 + xi_165 + xi_166 + - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_4 + xi_155 - xi_158 - xi_160 + xi_166 + xi_167 + - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_5 - xi_170 + xi_172 - xi_174 + xi_179 + xi_180 + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_6 + xi_170 - xi_172 + xi_174 + xi_180 + xi_181 + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_7 + xi_190 + xi_194 + xi_198 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_8 + xi_190 + xi_199 + xi_201 + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_9 + xi_194 + xi_199 + xi_203 + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_10 + xi_198 + xi_201 + xi_203 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_11 + xi_210 + xi_222 + xi_227 + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_12 + xi_222 + xi_228 + xi_229 + - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_13 + xi_234 + xi_237 + xi_242 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_14 + xi_234 + xi_243 + xi_244 + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_15 + xi_227 + xi_229 + xi_246 + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_16 + xi_210 + xi_228 + xi_246 + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_17 + xi_242 + xi_243 + xi_247 + - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_18 + xi_237 + xi_244 + xi_247 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void kernel_streamCollide(double * RESTRICT const _data_force, double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_0, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, double kT, double omega_bulk, double omega_even, double omega_odd, double omega_shear, uint32_t seed, uint32_t time_step) +{ + const double xi_40 = -omega_bulk; + const double xi_51 = -omega_shear; + const double xi_52 = xi_51 + 2.0; + const double xi_53 = xi_52*0.5; + const double xi_58 = xi_52*0.0833333333333333; + const double xi_63 = xi_52*0.166666666666667; + const double xi_73 = xi_52*0.25; + const double xi_78 = xi_52*0.0416666666666667; + const double xi_105 = 2.4494897427831779; + const double xi_129 = omega_odd*0.25; + const double xi_144 = omega_odd*0.0833333333333333; + const double xi_207 = omega_shear*0.25; + const double xi_222 = omega_odd*0.0416666666666667; + const double xi_224 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_133 = rr_0*0.166666666666667; + const double xi_197 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + for (int64_t ctr_0 = 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) + { + + double random_7_0; + double random_7_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 7, seed, random_7_0, random_7_1); + + + double random_6_0; + double random_6_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 6, seed, random_6_0, random_6_1); + + + double random_5_0; + double random_5_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 5, seed, random_5_0, random_5_1); + + + double random_4_0; + double random_4_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 4, seed, random_4_0, random_4_1); + + + double random_3_0; + double random_3_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1); + + + double random_2_0; + double random_2_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1); + + + double random_1_0; + double random_1_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1); + + + double random_0_0; + double random_0_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1); + + const double xi_0 = _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_1 = _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_2 = xi_0 + xi_1 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_3 = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_4 = xi_3 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + const double xi_5 = xi_4 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_6 = _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_7 = xi_6 + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_8 = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_9 = _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0] + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_10 = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + const double xi_12 = -_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_13 = -_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_16 = -_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_17 = -_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_21 = xi_20 + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_22 = -_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_23 = -_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + const double xi_24 = -_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + const double xi_29 = -_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + const double xi_30 = xi_24 + xi_28 + xi_29 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_31 = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_32 = xi_31 + xi_8 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_33 = xi_16 + xi_21 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_34 = xi_26 + _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_35 = xi_15 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_9 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_38 = xi_28 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_39 = xi_22 + xi_38 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_57 = 0.166666666666667*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_65 = 0.166666666666667*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_69 = 0.166666666666667*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_72 = 0.5*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_76 = 0.0833333333333333*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_80 = 0.0833333333333333*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_90 = 0.0833333333333333*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_108 = -_data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_109 = xi_108 + 3.0*_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_110 = omega_even*(xi_109 - 3.0*_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_111 = 2.0*_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_112 = xi_111 + 5.0*_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 5.0*_data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_113 = omega_even*(xi_109 + xi_112 - 2.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] - 2.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0] - 5.0*_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 5.0*_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 5.0*_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 5.0*_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]); + const double xi_116 = -_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_117 = xi_116 + xi_22; + const double xi_118 = -_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_121 = -_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_122 = xi_121 + xi_19 + xi_27; + const double xi_124 = 2.0*_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_125 = 2.0*_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_126 = 2.0*_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 2.0*_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_127 = omega_even*(xi_108 + xi_112 + xi_124 + xi_125 + xi_126 - 4.0*_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] - 4.0*_data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0] - 7.0*_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 7.0*_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 7.0*_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 7.0*_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 5.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + 5.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_128 = xi_116 + xi_23 + xi_38 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_130 = xi_128*xi_129; + const double xi_135 = random_5_1 - 0.5; + const double xi_140 = 2.0*_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_141 = 2.0*_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_142 = -2.0*_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + 2.0*_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_143 = -xi_140 + xi_141 + xi_142 + xi_25 + xi_4; + const double xi_145 = xi_143*xi_144; + const double xi_146 = random_3_0 - 0.5; + const double xi_151 = random_0_1 - 0.5; + const double xi_168 = xi_121 + xi_13 + xi_34 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_169 = xi_129*xi_168; + const double xi_170 = random_4_1 - 0.5; + const double xi_172 = xi_14 + xi_140 - xi_141 + xi_142 + xi_35 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_173 = xi_144*xi_172; + const double xi_174 = random_4_0 - 0.5; + const double xi_179 = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_180 = xi_117 + xi_179 + xi_29 + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_181 = xi_129*xi_180; + const double xi_184 = random_5_0 - 0.5; + const double xi_186 = -xi_124 - xi_125 + xi_126 + xi_30 + xi_6; + const double xi_187 = xi_144*xi_186; + const double xi_188 = random_3_1 - 0.5; + const double xi_195 = xi_127*0.0138888888888889; + const double xi_216 = xi_113*-0.00714285714285714; + const double xi_218 = xi_110*0.025; + const double xi_223 = xi_186*xi_222; + const double xi_225 = xi_180*xi_224; + const double xi_234 = xi_143*xi_222; + const double xi_235 = xi_128*xi_224; + const double xi_243 = xi_113*0.0178571428571429; + const double xi_249 = xi_168*xi_224; + const double xi_250 = xi_172*xi_222; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_8 + xi_9 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_11 = 1 / (rho); + const double xi_101 = kT*rho; + const double xi_102 = sqrt(xi_101*(-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0)); + const double xi_103 = xi_102*(random_6_0 - 0.5)*3.7416573867739413; + const double xi_104 = xi_102*(random_7_0 - 0.5)*5.4772255750516612; + const double xi_106 = xi_105*sqrt(xi_101*(-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0))*(random_2_1 - 0.5); + const double xi_107 = xi_102*(random_6_1 - 0.5)*8.3666002653407556; + const double xi_136 = sqrt(xi_101*(-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0)); + const double xi_137 = xi_136*1.4142135623730951; + const double xi_138 = xi_137*0.5; + const double xi_139 = xi_135*xi_138; + const double xi_147 = xi_105*xi_136; + const double xi_148 = xi_147*0.166666666666667; + const double xi_149 = xi_146*xi_148; + const double xi_150 = -xi_145 - xi_149; + const double xi_152 = sqrt(xi_101*(-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0)); + const double xi_153 = xi_152*0.5; + const double xi_154 = xi_151*xi_153; + const double xi_158 = xi_103*-0.119047619047619 + xi_127*-0.0198412698412698; + const double xi_160 = xi_152*(random_0_0 - 0.5)*1.7320508075688772; + const double xi_164 = xi_145 + xi_149; + const double xi_171 = xi_138*xi_170; + const double xi_175 = xi_148*xi_174; + const double xi_176 = xi_173 + xi_175; + const double xi_178 = -xi_173 - xi_175; + const double xi_185 = xi_138*xi_184; + const double xi_189 = xi_148*xi_188; + const double xi_190 = -xi_187 - xi_189; + const double xi_192 = xi_187 + xi_189; + const double xi_193 = xi_151*xi_152*0.25; + const double xi_196 = xi_103*0.0833333333333333; + const double xi_206 = xi_153*(random_1_0 - 0.5); + const double xi_215 = xi_153*(random_2_0 - 0.5); + const double xi_219 = xi_107*-0.0142857142857143; + const double xi_220 = xi_104*0.05; + const double xi_226 = xi_147*0.0833333333333333; + const double xi_227 = xi_188*xi_226; + const double xi_228 = xi_137*0.25; + const double xi_229 = xi_184*xi_228; + const double xi_231 = xi_103*-0.0238095238095238 + xi_127*-0.00396825396825397; + const double xi_236 = xi_146*xi_226; + const double xi_237 = xi_135*xi_228; + const double xi_241 = -xi_193; + const double xi_244 = xi_107*0.0357142857142857; + const double xi_246 = xi_153*(random_1_1 - 0.5); + const double xi_251 = xi_170*xi_228; + const double xi_252 = xi_174*xi_226; + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_41 = u_0*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_42 = xi_41*0.333333333333333; + const double xi_48 = -xi_42; + const double xi_114 = rho*(u_0*u_0); + const double xi_165 = rho*u_0; + const double xi_166 = -vel0Term + xi_165 + xi_32; + const double xi_167 = xi_133*xi_166; + const double xi_202 = xi_166*xi_197; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_43 = u_1*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_44 = xi_43*0.333333333333333; + const double xi_49 = -xi_44; + const double xi_71 = u_1*0.5; + const double xi_74 = xi_73*(u_0*xi_72 + xi_71*_data_force_20_30_10[_stride_force_0*ctr_0]); + const double xi_75 = -xi_74; + const double xi_119 = rho*(u_1*u_1); + const double xi_120 = xi_118 + xi_119 + xi_20; + const double xi_131 = rho*u_1; + const double xi_132 = -vel1Term + xi_118 + xi_131 + xi_37 + _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_134 = xi_132*xi_133; + const double xi_198 = xi_132*xi_197; + const double u_2 = xi_11*(vel2Term + xi_27 + xi_30 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]); + const double xi_45 = u_2*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_46 = xi_45*0.333333333333333; + const double xi_47 = (xi_40 + 2.0)*(xi_42 + xi_44 + xi_46); + const double xi_50 = xi_45*0.666666666666667 + xi_48 + xi_49; + const double xi_54 = -xi_46; + const double xi_55 = xi_43*0.666666666666667 + xi_48 + xi_54; + const double xi_56 = xi_41*0.666666666666667 + xi_49 + xi_54; + const double xi_59 = xi_50*xi_58; + const double xi_60 = -xi_59; + const double xi_61 = xi_56*xi_58; + const double xi_62 = -xi_61; + const double xi_64 = xi_55*xi_63 + xi_60 + xi_62; + const double xi_66 = xi_55*xi_58; + const double xi_67 = -xi_66; + const double xi_68 = xi_56*xi_63 + xi_60 + xi_67; + const double xi_70 = xi_50*xi_63 + xi_62 + xi_67; + const double xi_77 = xi_61 - xi_76; + const double xi_79 = -xi_50*xi_78; + const double xi_81 = xi_47*0.125; + const double xi_82 = xi_66 + xi_81; + const double xi_83 = xi_80 + xi_82; + const double xi_84 = xi_79 + xi_83; + const double xi_85 = xi_61 + xi_76; + const double xi_86 = -xi_80 + xi_82; + const double xi_87 = xi_79 + xi_86; + const double xi_88 = xi_73*(u_2*xi_72 + xi_71*_data_force_20_32_10[_stride_force_0*ctr_0]); + const double xi_89 = -xi_56*xi_78; + const double xi_91 = xi_59 + xi_90; + const double xi_92 = xi_89 + xi_91; + const double xi_93 = -xi_88; + const double xi_94 = xi_73*(u_0*0.5*_data_force_20_32_10[_stride_force_0*ctr_0] + u_2*0.5*_data_force_20_30_10[_stride_force_0*ctr_0]); + const double xi_95 = -xi_94; + const double xi_96 = -xi_55*xi_78; + const double xi_97 = xi_81 + xi_91 + xi_96; + const double xi_98 = xi_59 - xi_90; + const double xi_99 = xi_89 + xi_98; + const double xi_100 = xi_81 + xi_96 + xi_98; + const double xi_115 = rho*(u_2*u_2); + const double xi_123 = omega_bulk*(xi_114 + xi_115 + xi_117 + xi_120 + xi_122 + xi_24 + xi_28 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]); + const double xi_155 = -xi_115 + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_156 = omega_shear*(xi_1 + xi_120 + xi_155 + xi_18 + xi_23 + xi_31 - _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]); + const double xi_157 = xi_156*0.125; + const double xi_159 = omega_shear*(xi_111 + xi_114*2.0 + xi_118 - xi_119 + xi_122 + xi_155 + xi_20 - 2.0*_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 2.0*_data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_161 = xi_159*-0.0416666666666667 + xi_160*-0.166666666666667; + const double xi_162 = xi_104*-0.1 + xi_110*-0.05 + xi_161; + const double xi_163 = xi_107*0.0285714285714286 + xi_113*0.0142857142857143 + xi_154 + xi_157 + xi_158 + xi_162; + const double xi_177 = xi_107*-0.0714285714285714 + xi_113*-0.0357142857142857 + xi_158 + xi_159*0.0833333333333333 + xi_160*0.333333333333333; + const double xi_182 = rho*u_2 - vel2Term + xi_10 + xi_116 + xi_121 + xi_179 + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_183 = xi_133*xi_182; + const double xi_191 = xi_103*0.0952380952380952 + xi_107*-0.0428571428571429 + xi_113*-0.0214285714285714 + xi_127*0.0158730158730159 - xi_154 - xi_157 + xi_162; + const double xi_194 = xi_156*0.0625; + const double xi_199 = xi_106*0.0833333333333333 + xi_123*0.0416666666666667; + const double xi_200 = xi_198 + xi_199; + const double xi_201 = xi_164 + xi_193 + xi_194 + xi_195 + xi_196 + xi_200; + const double xi_203 = xi_159*0.0208333333333333 + xi_160*0.0833333333333333; + const double xi_204 = -xi_202 + xi_203; + const double xi_205 = xi_178 + xi_204; + const double xi_211 = xi_202 + xi_203; + const double xi_212 = xi_176 + xi_211; + const double xi_213 = -xi_198 + xi_199; + const double xi_214 = xi_150 + xi_193 + xi_194 + xi_195 + xi_196 + xi_213; + const double xi_230 = xi_182*xi_197; + const double xi_232 = xi_230 + xi_231; + const double xi_233 = -xi_223 + xi_225 - xi_227 + xi_229 + xi_232; + const double xi_238 = xi_200 - xi_234 + xi_235 - xi_236 + xi_237; + const double xi_239 = xi_213 + xi_234 - xi_235 + xi_236 - xi_237; + const double xi_242 = -xi_194; + const double xi_245 = xi_192 + xi_199 + xi_232 + xi_241 + xi_242 + xi_243 + xi_244; + const double xi_253 = xi_204 - xi_249 + xi_250 - xi_251 + xi_252; + const double xi_255 = xi_211 + xi_249 - xi_250 + xi_251 - xi_252; + const double xi_256 = -xi_230 + xi_231; + const double xi_257 = xi_223 - xi_225 + xi_227 - xi_229 + xi_256; + const double xi_258 = xi_190 + xi_199 + xi_241 + xi_242 + xi_243 + xi_244 + xi_256; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_208 = xi_207*(-p_1 + u_0*xi_131); + const double xi_209 = -xi_206 - xi_208; + const double xi_210 = xi_206 + xi_208; + const double p_2 = xi_36; + const double xi_247 = xi_207*(-p_2 + u_2*xi_165); + const double xi_248 = -xi_246 - xi_247; + const double xi_254 = xi_246 + xi_247; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_217 = xi_207*(-p_5 + u_2*xi_131); + const double xi_221 = xi_161 + xi_215 + xi_216 + xi_217 + xi_218 + xi_219 + xi_220; + const double xi_240 = xi_161 - xi_215 + xi_216 - xi_217 + xi_218 + xi_219 + xi_220; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_3 + xi_7 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double forceTerm_0 = xi_47*-1.5 - xi_50*xi_53 - xi_53*xi_55 - xi_53*xi_56; + const double forceTerm_1 = xi_57 + xi_64; + const double forceTerm_2 = -xi_57 + xi_64; + const double forceTerm_3 = -xi_65 + xi_68; + const double forceTerm_4 = xi_65 + xi_68; + const double forceTerm_5 = xi_69 + xi_70; + const double forceTerm_6 = -xi_69 + xi_70; + const double forceTerm_7 = xi_75 + xi_77 + xi_84; + const double forceTerm_8 = xi_74 + xi_84 + xi_85; + const double forceTerm_9 = xi_74 + xi_77 + xi_87; + const double forceTerm_10 = xi_75 + xi_85 + xi_87; + const double forceTerm_11 = xi_83 + xi_88 + xi_92; + const double forceTerm_12 = xi_86 + xi_92 + xi_93; + const double forceTerm_13 = xi_77 + xi_95 + xi_97; + const double forceTerm_14 = xi_85 + xi_94 + xi_97; + const double forceTerm_15 = xi_83 + xi_93 + xi_99; + const double forceTerm_16 = xi_86 + xi_88 + xi_99; + const double forceTerm_17 = xi_100 + xi_77 + xi_94; + const double forceTerm_18 = xi_100 + xi_85 + xi_95; + _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_0 + xi_103*0.142857142857143 + xi_104*0.2 - xi_106 + xi_107*0.0857142857142857 + xi_110*0.1 + xi_113*0.0428571428571429 + xi_123*-0.5 + xi_127*0.0238095238095238 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_1 - xi_130 + xi_134 - xi_139 + xi_150 + xi_163 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_2 + xi_130 - xi_134 + xi_139 + xi_163 + xi_164 + _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_3 - xi_167 + xi_169 + xi_171 + xi_176 + xi_177 + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_4 + xi_167 - xi_169 - xi_171 + xi_177 + xi_178 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_5 - xi_181 + xi_183 - xi_185 + xi_190 + xi_191 + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_6 + xi_181 - xi_183 + xi_185 + xi_191 + xi_192 + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_7 + xi_201 + xi_205 + xi_209 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_8 + xi_201 + xi_210 + xi_212 + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_9 + xi_205 + xi_210 + xi_214 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_10 + xi_209 + xi_212 + xi_214 + _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_11 + xi_221 + xi_233 + xi_238 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_12 + xi_233 + xi_239 + xi_240 + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_13 + xi_245 + xi_248 + xi_253 + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_14 + xi_245 + xi_254 + xi_255 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_15 + xi_238 + xi_240 + xi_257 + _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_16 + xi_221 + xi_239 + xi_257 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_17 + xi_253 + xi_254 + xi_258 + _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_18 + xi_248 + xi_255 + xi_258 + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_kernel_streamCollide namespace internal_kernel_collide { -static FUNC_PREFIX void -kernel_collide(double *RESTRICT const _data_force, double *RESTRICT _data_pdfs, - int64_t const _size_force_0, int64_t const _size_force_1, - int64_t const _size_force_2, int64_t const _stride_force_0, - int64_t const _stride_force_1, int64_t const _stride_force_2, - int64_t const _stride_force_3, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, uint32_t block_offset_0, - uint32_t block_offset_1, uint32_t block_offset_2, double kT, - double omega_bulk, double omega_even, double omega_odd, - double omega_shear, uint32_t seed, uint32_t time_step) { - const double xi_25 = -omega_bulk; - const double xi_36 = -omega_shear; - const double xi_37 = xi_36 + 2.0; - const double xi_38 = xi_37 * 0.5; - const double xi_43 = xi_37 * 0.0833333333333333; - const double xi_48 = xi_37 * 0.166666666666667; - const double xi_58 = xi_37 * 0.25; - const double xi_63 = xi_37 * 0.0416666666666667; - const double xi_90 = 2.4494897427831779; - const double xi_115 = omega_odd * 0.25; - const double xi_131 = omega_odd * 0.0833333333333333; - const double xi_196 = omega_shear * 0.25; - const double xi_211 = omega_odd * 0.0416666666666667; - const double xi_213 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_120 = rr_0 * 0.166666666666667; - const double xi_186 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) { - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_313 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_316 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_311 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_312 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_314 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_315 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_35 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_36 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_317 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_318 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) { - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_313; - double *RESTRICT _data_pdfs_20_316_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_316; - double *RESTRICT _data_pdfs_20_311_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_311; - double *RESTRICT _data_pdfs_20_310_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_20_38_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_20_312_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_312; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_20_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_314; - double *RESTRICT _data_pdfs_20_315_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_315; - double *RESTRICT _data_pdfs_20_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_35; - double *RESTRICT _data_pdfs_20_37_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_32_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_32; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_31_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_20_39_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_36; - double *RESTRICT _data_pdfs_20_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_317; - double *RESTRICT _data_pdfs_20_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_318; - for (int64_t ctr_0 = 0; ctr_0 < _size_force_0; ctr_0 += 1) { - const double xi_248 = _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_249 = _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_250 = _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0]; - const double xi_251 = _data_pdfs_20_313_10[_stride_pdfs_0 * ctr_0]; - const double xi_252 = _data_pdfs_20_316_10[_stride_pdfs_0 * ctr_0]; - const double xi_253 = _data_pdfs_20_311_10[_stride_pdfs_0 * ctr_0]; - const double xi_254 = _data_pdfs_20_310_10[_stride_pdfs_0 * ctr_0]; - const double xi_255 = _data_pdfs_20_38_10[_stride_pdfs_0 * ctr_0]; - const double xi_256 = _data_pdfs_20_312_10[_stride_pdfs_0 * ctr_0]; - const double xi_257 = _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0]; - const double xi_258 = _data_pdfs_20_314_10[_stride_pdfs_0 * ctr_0]; - const double xi_259 = _data_pdfs_20_315_10[_stride_pdfs_0 * ctr_0]; - const double xi_260 = _data_pdfs_20_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_261 = _data_pdfs_20_37_10[_stride_pdfs_0 * ctr_0]; - const double xi_262 = _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_263 = _data_pdfs_20_32_10[_stride_pdfs_0 * ctr_0]; - const double xi_264 = _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_265 = _data_pdfs_20_31_10[_stride_pdfs_0 * ctr_0]; - const double xi_266 = _data_pdfs_20_39_10[_stride_pdfs_0 * ctr_0]; - const double xi_267 = _data_pdfs_20_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_268 = _data_pdfs_20_317_10[_stride_pdfs_0 * ctr_0]; - const double xi_269 = _data_pdfs_20_318_10[_stride_pdfs_0 * ctr_0]; - - double random_7_0; - double random_7_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 7, seed, - random_7_0, random_7_1); - - double random_6_0; - double random_6_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 6, seed, - random_6_0, random_6_1); - - double random_5_0; - double random_5_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 5, seed, - random_5_0, random_5_1); - - double random_4_0; - double random_4_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 4, seed, - random_4_0, random_4_1); - - double random_3_0; - double random_3_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, - random_3_0, random_3_1); - - double random_2_0; - double random_2_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, - random_2_0, random_2_1); - - double random_1_0; - double random_1_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, - random_1_0, random_1_1); - - double random_0_0; - double random_0_1; - philox_double2(time_step, block_offset_0 + ctr_0, - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, - random_0_0, random_0_1); - - const double xi_0 = xi_258 + xi_269; - const double xi_1 = xi_0 + xi_257; - const double xi_2 = xi_253 + xi_259 + xi_265; - const double xi_3 = xi_256 + xi_260; - const double xi_4 = xi_250 + xi_266; - const double xi_5 = xi_252 + xi_263; - const double xi_6 = xi_267 + xi_268; - const double xi_8 = -xi_266; - const double xi_9 = -xi_261 + xi_8; - const double xi_10 = -xi_268; - const double xi_11 = -xi_251; - const double xi_12 = -xi_250; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -xi_263; - const double xi_15 = -xi_254; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -xi_252; - const double xi_18 = -xi_256; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = -xi_269; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -xi_259; - const double xi_23 = -xi_267; - const double xi_24 = xi_17 + xi_22 + xi_23 + xi_253; - const double xi_42 = xi_248 * 0.166666666666667; - const double xi_50 = xi_249 * 0.166666666666667; - const double xi_54 = xi_264 * 0.166666666666667; - const double xi_57 = xi_248 * 0.5; - const double xi_61 = xi_249 * 0.0833333333333333; - const double xi_65 = xi_248 * 0.0833333333333333; - const double xi_75 = xi_264 * 0.0833333333333333; - const double xi_93 = -xi_262; - const double xi_94 = xi_260 * 3.0 + xi_267 * 3.0 + xi_93; - const double xi_95 = - omega_even * (xi_252 * -3.0 + xi_253 * -3.0 + xi_256 * -3.0 + - xi_259 * -3.0 + xi_263 * 3.0 + xi_265 * 3.0 + xi_94); - const double xi_96 = - xi_252 * 2.0 + xi_253 * 2.0 + xi_256 * 2.0 + xi_259 * 2.0; - const double xi_97 = xi_250 * 5.0 + xi_257 * 5.0 + xi_96; - const double xi_98 = - omega_even * - (xi_251 * -5.0 + xi_258 * -5.0 + xi_263 * -2.0 + xi_265 * -2.0 + - xi_268 * -5.0 + xi_269 * -5.0 + xi_94 + xi_97); - const double xi_101 = -xi_253; - const double xi_102 = xi_101 + xi_18; - const double xi_103 = -xi_255; - const double xi_106 = -xi_258; - const double xi_107 = xi_106 + xi_11 + xi_15 + xi_21; - const double xi_109 = xi_251 * 2.0; - const double xi_110 = xi_258 * 2.0; - const double xi_111 = xi_268 * 2.0 + xi_269 * 2.0; - const double xi_112 = - omega_even * - (xi_109 + xi_110 + xi_111 + xi_254 * -7.0 + xi_255 * -7.0 + - xi_260 * -4.0 + xi_261 * -7.0 + xi_263 * 5.0 + xi_265 * 5.0 + - xi_266 * -7.0 + xi_267 * -4.0 + xi_93 + xi_97); - const double xi_113 = xi_101 + xi_256; - const double xi_114 = xi_113 + xi_14 + xi_22 + xi_252 + xi_265; - const double xi_116 = xi_114 * xi_115; - const double xi_118 = xi_103 + xi_254; - const double xi_122 = random_5_1 - 0.5; - const double xi_127 = xi_261 * 2.0; - const double xi_128 = xi_254 * 2.0; - const double xi_129 = xi_255 * -2.0 + xi_266 * 2.0; - const double xi_130 = -xi_127 + xi_128 + xi_129 + xi_14 + xi_19 + xi_2; - const double xi_132 = xi_130 * xi_131; - const double xi_133 = random_3_0 - 0.5; - const double xi_138 = random_0_1 - 0.5; - const double xi_142 = xi_251 + xi_268; - const double xi_156 = xi_106 + xi_251; - const double xi_157 = xi_12 + xi_156 + xi_20 + xi_257 + xi_268; - const double xi_158 = xi_115 * xi_157; - const double xi_159 = random_4_1 - 0.5; - const double xi_161 = xi_1 + xi_127 - xi_128 + xi_129 + xi_13; - const double xi_162 = xi_131 * xi_161; - const double xi_163 = random_4_0 - 0.5; - const double xi_168 = xi_252 + xi_259; - const double xi_169 = xi_102 + xi_168 + xi_23 + xi_260; - const double xi_170 = xi_115 * xi_169; - const double xi_173 = random_5_0 - 0.5; - const double xi_175 = -xi_109 - xi_110 + xi_111 + xi_24 + xi_3; - const double xi_176 = xi_131 * xi_175; - const double xi_177 = random_3_1 - 0.5; - const double xi_184 = xi_112 * 0.0138888888888889; - const double xi_205 = xi_98 * -0.00714285714285714; - const double xi_207 = xi_95 * 0.025; - const double xi_212 = xi_175 * xi_211; - const double xi_214 = xi_169 * xi_213; - const double xi_223 = xi_130 * xi_211; - const double xi_224 = xi_114 * xi_213; - const double xi_232 = xi_98 * 0.0178571428571429; - const double xi_238 = xi_157 * xi_213; - const double xi_239 = xi_161 * xi_211; - const double vel0Term = xi_1 + xi_254 + xi_255; - const double vel1Term = xi_2 + xi_261; - const double vel2Term = xi_251 + xi_3; - const double rho = - vel0Term + vel1Term + vel2Term + xi_262 + xi_4 + xi_5 + xi_6; - const double xi_7 = 1 / (rho); - const double xi_86 = kT * rho; - const double xi_87 = - sqrt(xi_86 * (-((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0)); - const double xi_88 = xi_87 * (random_6_0 - 0.5) * 3.7416573867739413; - const double xi_89 = xi_87 * (random_7_0 - 0.5) * 5.4772255750516612; - const double xi_91 = - xi_90 * sqrt(xi_86 * (-((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0)) * - (random_2_1 - 0.5); - const double xi_92 = xi_87 * (random_6_1 - 0.5) * 8.3666002653407556; - const double xi_123 = - sqrt(xi_86 * (-((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0)); - const double xi_124 = xi_123 * 1.4142135623730951; - const double xi_125 = xi_124 * 0.5; - const double xi_126 = xi_122 * xi_125; - const double xi_134 = xi_123 * xi_90; - const double xi_135 = xi_134 * 0.166666666666667; - const double xi_136 = xi_133 * xi_135; - const double xi_137 = -xi_132 - xi_136; - const double xi_139 = - sqrt(xi_86 * (-((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0)); - const double xi_140 = xi_139 * 0.5; - const double xi_141 = xi_138 * xi_140; - const double xi_146 = - xi_112 * -0.0198412698412698 + xi_88 * -0.119047619047619; - const double xi_148 = xi_139 * (random_0_0 - 0.5) * 1.7320508075688772; - const double xi_152 = xi_132 + xi_136; - const double xi_160 = xi_125 * xi_159; - const double xi_164 = xi_135 * xi_163; - const double xi_165 = xi_162 + xi_164; - const double xi_167 = -xi_162 - xi_164; - const double xi_174 = xi_125 * xi_173; - const double xi_178 = xi_135 * xi_177; - const double xi_179 = -xi_176 - xi_178; - const double xi_181 = xi_176 + xi_178; - const double xi_182 = xi_138 * xi_139 * 0.25; - const double xi_185 = xi_88 * 0.0833333333333333; - const double xi_195 = xi_140 * (random_1_0 - 0.5); - const double xi_204 = xi_140 * (random_2_0 - 0.5); - const double xi_208 = xi_92 * -0.0142857142857143; - const double xi_209 = xi_89 * 0.05; - const double xi_215 = xi_134 * 0.0833333333333333; - const double xi_216 = xi_177 * xi_215; - const double xi_217 = xi_124 * 0.25; - const double xi_218 = xi_173 * xi_217; - const double xi_220 = - xi_112 * -0.00396825396825397 + xi_88 * -0.0238095238095238; - const double xi_225 = xi_133 * xi_215; - const double xi_226 = xi_122 * xi_217; - const double xi_230 = -xi_182; - const double xi_233 = xi_92 * 0.0357142857142857; - const double xi_235 = xi_140 * (random_1_1 - 0.5); - const double xi_240 = xi_159 * xi_217; - const double xi_241 = xi_163 * xi_215; - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_26 = u_0 * xi_249; - const double xi_27 = xi_26 * 0.333333333333333; - const double xi_33 = -xi_27; - const double xi_99 = rho * (u_0 * u_0); - const double xi_153 = rho * u_0; - const double xi_154 = -vel0Term + xi_142 + xi_153 + xi_261 + xi_4; - const double xi_155 = xi_120 * xi_154; - const double xi_191 = xi_154 * xi_186; - const double u_1 = xi_7 * (vel1Term + xi_16 + xi_19 + xi_255 + xi_8); - const double xi_28 = u_1 * xi_248; - const double xi_29 = xi_28 * 0.333333333333333; - const double xi_34 = -xi_29; - const double xi_56 = u_1 * 0.5; - const double xi_59 = xi_58 * (u_0 * xi_57 + xi_249 * xi_56); - const double xi_60 = -xi_59; - const double xi_104 = rho * (u_1 * u_1); - const double xi_105 = xi_103 + xi_104 + xi_9; - const double xi_117 = rho * u_1; - const double xi_119 = - -vel1Term + xi_117 + xi_118 + xi_256 + xi_266 + xi_5; - const double xi_121 = xi_119 * xi_120; - const double xi_187 = xi_119 * xi_186; - const double xi_197 = xi_196 * (u_0 * xi_117 + xi_118 + xi_261 + xi_8); - const double xi_198 = -xi_195 - xi_197; - const double xi_199 = xi_195 + xi_197; - const double u_2 = xi_7 * (vel2Term + xi_21 + xi_24 + xi_258); - const double xi_30 = u_2 * xi_264; - const double xi_31 = xi_30 * 0.333333333333333; - const double xi_32 = (xi_25 + 2.0) * (xi_27 + xi_29 + xi_31); - const double xi_35 = xi_30 * 0.666666666666667 + xi_33 + xi_34; - const double xi_39 = -xi_31; - const double xi_40 = xi_28 * 0.666666666666667 + xi_33 + xi_39; - const double xi_41 = xi_26 * 0.666666666666667 + xi_34 + xi_39; - const double xi_44 = xi_35 * xi_43; - const double xi_45 = -xi_44; - const double xi_46 = xi_41 * xi_43; - const double xi_47 = -xi_46; - const double xi_49 = xi_40 * xi_48 + xi_45 + xi_47; - const double xi_51 = xi_40 * xi_43; - const double xi_52 = -xi_51; - const double xi_53 = xi_41 * xi_48 + xi_45 + xi_52; - const double xi_55 = xi_35 * xi_48 + xi_47 + xi_52; - const double xi_62 = xi_46 - xi_61; - const double xi_64 = -xi_35 * xi_63; - const double xi_66 = xi_32 * 0.125; - const double xi_67 = xi_51 + xi_66; - const double xi_68 = xi_65 + xi_67; - const double xi_69 = xi_64 + xi_68; - const double xi_70 = xi_46 + xi_61; - const double xi_71 = -xi_65 + xi_67; - const double xi_72 = xi_64 + xi_71; - const double xi_73 = xi_58 * (u_2 * xi_57 + xi_264 * xi_56); - const double xi_74 = -xi_41 * xi_63; - const double xi_76 = xi_44 + xi_75; - const double xi_77 = xi_74 + xi_76; - const double xi_78 = -xi_73; - const double xi_79 = xi_58 * (u_0 * xi_264 * 0.5 + u_2 * xi_249 * 0.5); - const double xi_80 = -xi_79; - const double xi_81 = -xi_40 * xi_63; - const double xi_82 = xi_66 + xi_76 + xi_81; - const double xi_83 = xi_44 - xi_75; - const double xi_84 = xi_74 + xi_83; - const double xi_85 = xi_66 + xi_81 + xi_83; - const double xi_100 = rho * (u_2 * u_2); - const double xi_108 = omega_bulk * (xi_100 + xi_102 + xi_105 + xi_107 + - xi_17 + xi_22 + xi_262 + xi_99); - const double xi_143 = -xi_100 + xi_260 + xi_267; - const double xi_144 = - omega_shear * (xi_0 + xi_105 + xi_142 + xi_143 + xi_16 - xi_265); - const double xi_145 = xi_144 * 0.125; - const double xi_147 = - omega_shear * - (xi_103 - xi_104 + xi_107 + xi_143 + xi_250 * -2.0 + xi_257 * -2.0 + - xi_263 + xi_265 + xi_9 + xi_96 + xi_99 * 2.0); - const double xi_149 = - xi_147 * -0.0416666666666667 + xi_148 * -0.166666666666667; - const double xi_150 = xi_149 + xi_89 * -0.1 + xi_95 * -0.05; - const double xi_151 = xi_141 + xi_145 + xi_146 + xi_150 + - xi_92 * 0.0285714285714286 + - xi_98 * 0.0142857142857143; - const double xi_166 = - xi_146 + xi_147 * 0.0833333333333333 + xi_148 * 0.333333333333333 + - xi_92 * -0.0714285714285714 + xi_98 * -0.0357142857142857; - const double xi_171 = - rho * u_2 - vel2Term + xi_101 + xi_106 + xi_168 + xi_269 + xi_6; - const double xi_172 = xi_120 * xi_171; - const double xi_180 = xi_112 * 0.0158730158730159 - xi_141 - xi_145 + - xi_150 + xi_88 * 0.0952380952380952 + - xi_92 * -0.0428571428571429 + - xi_98 * -0.0214285714285714; - const double xi_183 = xi_144 * 0.0625; - const double xi_188 = - xi_108 * 0.0416666666666667 + xi_91 * 0.0833333333333333; - const double xi_189 = xi_187 + xi_188; - const double xi_190 = - xi_152 + xi_182 + xi_183 + xi_184 + xi_185 + xi_189; - const double xi_192 = - xi_147 * 0.0208333333333333 + xi_148 * 0.0833333333333333; - const double xi_193 = -xi_191 + xi_192; - const double xi_194 = xi_167 + xi_193; - const double xi_200 = xi_191 + xi_192; - const double xi_201 = xi_165 + xi_200; - const double xi_202 = -xi_187 + xi_188; - const double xi_203 = - xi_137 + xi_182 + xi_183 + xi_184 + xi_185 + xi_202; - const double xi_206 = xi_196 * (u_2 * xi_117 + xi_113 + xi_17 + xi_259); - const double xi_210 = - xi_149 + xi_204 + xi_205 + xi_206 + xi_207 + xi_208 + xi_209; - const double xi_219 = xi_171 * xi_186; - const double xi_221 = xi_219 + xi_220; - const double xi_222 = -xi_212 + xi_214 - xi_216 + xi_218 + xi_221; - const double xi_227 = xi_189 - xi_223 + xi_224 - xi_225 + xi_226; - const double xi_228 = xi_202 + xi_223 - xi_224 + xi_225 - xi_226; - const double xi_229 = - xi_149 - xi_204 + xi_205 - xi_206 + xi_207 + xi_208 + xi_209; - const double xi_231 = -xi_183; - const double xi_234 = - xi_181 + xi_188 + xi_221 + xi_230 + xi_231 + xi_232 + xi_233; - const double xi_236 = xi_196 * (u_2 * xi_153 + xi_10 + xi_156 + xi_269); - const double xi_237 = -xi_235 - xi_236; - const double xi_242 = xi_193 - xi_238 + xi_239 - xi_240 + xi_241; - const double xi_243 = xi_235 + xi_236; - const double xi_244 = xi_200 + xi_238 - xi_239 + xi_240 - xi_241; - const double xi_245 = -xi_219 + xi_220; - const double xi_246 = xi_212 - xi_214 + xi_216 - xi_218 + xi_245; - const double xi_247 = - xi_179 + xi_188 + xi_230 + xi_231 + xi_232 + xi_233 + xi_245; - const double forceTerm_0 = - xi_32 * -1.5 - xi_35 * xi_38 - xi_38 * xi_40 - xi_38 * xi_41; - const double forceTerm_1 = xi_42 + xi_49; - const double forceTerm_2 = -xi_42 + xi_49; - const double forceTerm_3 = -xi_50 + xi_53; - const double forceTerm_4 = xi_50 + xi_53; - const double forceTerm_5 = xi_54 + xi_55; - const double forceTerm_6 = -xi_54 + xi_55; - const double forceTerm_7 = xi_60 + xi_62 + xi_69; - const double forceTerm_8 = xi_59 + xi_69 + xi_70; - const double forceTerm_9 = xi_59 + xi_62 + xi_72; - const double forceTerm_10 = xi_60 + xi_70 + xi_72; - const double forceTerm_11 = xi_68 + xi_73 + xi_77; - const double forceTerm_12 = xi_71 + xi_77 + xi_78; - const double forceTerm_13 = xi_62 + xi_80 + xi_82; - const double forceTerm_14 = xi_70 + xi_79 + xi_82; - const double forceTerm_15 = xi_68 + xi_78 + xi_84; - const double forceTerm_16 = xi_71 + xi_73 + xi_84; - const double forceTerm_17 = xi_62 + xi_79 + xi_85; - const double forceTerm_18 = xi_70 + xi_80 + xi_85; - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0] = - forceTerm_0 + xi_108 * -0.5 + xi_112 * 0.0238095238095238 + xi_262 + - xi_88 * 0.142857142857143 + xi_89 * 0.2 - xi_91 + - xi_92 * 0.0857142857142857 + xi_95 * 0.1 + - xi_98 * 0.0428571428571429; - _data_pdfs_20_31_10[_stride_pdfs_0 * ctr_0] = - forceTerm_1 - xi_116 + xi_121 - xi_126 + xi_137 + xi_151 + xi_265; - _data_pdfs_20_32_10[_stride_pdfs_0 * ctr_0] = - forceTerm_2 + xi_116 - xi_121 + xi_126 + xi_151 + xi_152 + xi_263; - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0] = - forceTerm_3 - xi_155 + xi_158 + xi_160 + xi_165 + xi_166 + xi_250; - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0] = - forceTerm_4 + xi_155 - xi_158 - xi_160 + xi_166 + xi_167 + xi_257; - _data_pdfs_20_35_10[_stride_pdfs_0 * ctr_0] = - forceTerm_5 - xi_170 + xi_172 - xi_174 + xi_179 + xi_180 + xi_260; - _data_pdfs_20_36_10[_stride_pdfs_0 * ctr_0] = - forceTerm_6 + xi_170 - xi_172 + xi_174 + xi_180 + xi_181 + xi_267; - _data_pdfs_20_37_10[_stride_pdfs_0 * ctr_0] = - forceTerm_7 + xi_190 + xi_194 + xi_198 + xi_261; - _data_pdfs_20_38_10[_stride_pdfs_0 * ctr_0] = - forceTerm_8 + xi_190 + xi_199 + xi_201 + xi_255; - _data_pdfs_20_39_10[_stride_pdfs_0 * ctr_0] = - forceTerm_9 + xi_194 + xi_199 + xi_203 + xi_266; - _data_pdfs_20_310_10[_stride_pdfs_0 * ctr_0] = - forceTerm_10 + xi_198 + xi_201 + xi_203 + xi_254; - _data_pdfs_20_311_10[_stride_pdfs_0 * ctr_0] = - forceTerm_11 + xi_210 + xi_222 + xi_227 + xi_253; - _data_pdfs_20_312_10[_stride_pdfs_0 * ctr_0] = - forceTerm_12 + xi_222 + xi_228 + xi_229 + xi_256; - _data_pdfs_20_313_10[_stride_pdfs_0 * ctr_0] = - forceTerm_13 + xi_234 + xi_237 + xi_242 + xi_251; - _data_pdfs_20_314_10[_stride_pdfs_0 * ctr_0] = - forceTerm_14 + xi_234 + xi_243 + xi_244 + xi_258; - _data_pdfs_20_315_10[_stride_pdfs_0 * ctr_0] = - forceTerm_15 + xi_227 + xi_229 + xi_246 + xi_259; - _data_pdfs_20_316_10[_stride_pdfs_0 * ctr_0] = - forceTerm_16 + xi_210 + xi_228 + xi_246 + xi_252; - _data_pdfs_20_317_10[_stride_pdfs_0 * ctr_0] = - forceTerm_17 + xi_242 + xi_243 + xi_247 + xi_268; - _data_pdfs_20_318_10[_stride_pdfs_0 * ctr_0] = - forceTerm_18 + xi_237 + xi_244 + xi_247 + xi_269; +static FUNC_PREFIX void kernel_collide(double * RESTRICT const _data_force, double * RESTRICT _data_pdfs, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_0, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, double kT, double omega_bulk, double omega_even, double omega_odd, double omega_shear, uint32_t seed, uint32_t time_step) +{ + const double xi_40 = -omega_bulk; + const double xi_51 = -omega_shear; + const double xi_52 = xi_51 + 2.0; + const double xi_53 = xi_52*0.5; + const double xi_58 = xi_52*0.0833333333333333; + const double xi_63 = xi_52*0.166666666666667; + const double xi_73 = xi_52*0.25; + const double xi_78 = xi_52*0.0416666666666667; + const double xi_105 = 2.4494897427831779; + const double xi_129 = omega_odd*0.25; + const double xi_144 = omega_odd*0.0833333333333333; + const double xi_207 = omega_shear*0.25; + const double xi_222 = omega_odd*0.0416666666666667; + const double xi_224 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_133 = rr_0*0.166666666666667; + const double xi_197 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_20_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_pdfs_20_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + 16*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_38_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_20_31_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_35; + double * RESTRICT _data_pdfs_20_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_318; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_20_311_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_311; + double * RESTRICT _data_pdfs_20_310_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_20_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_36; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_37_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_20_315_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_315; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_pdfs_20_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_314; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_39_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_313; + double * RESTRICT _data_pdfs_20_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_317; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_pdfs_20_32_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_20_312_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_312; + double * RESTRICT _data_pdfs_20_316_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_316; + for (int64_t ctr_0 = 0; ctr_0 < _size_force_0; ctr_0 += 1) + { + const double xi_259 = _data_pdfs_20_38_10[_stride_pdfs_0*ctr_0]; + const double xi_260 = _data_pdfs_20_31_10[_stride_pdfs_0*ctr_0]; + const double xi_261 = _data_pdfs_20_35_10[_stride_pdfs_0*ctr_0]; + const double xi_262 = _data_pdfs_20_318_10[_stride_pdfs_0*ctr_0]; + const double xi_263 = _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_264 = _data_pdfs_20_311_10[_stride_pdfs_0*ctr_0]; + const double xi_265 = _data_pdfs_20_310_10[_stride_pdfs_0*ctr_0]; + const double xi_266 = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0]; + const double xi_267 = _data_pdfs_20_36_10[_stride_pdfs_0*ctr_0]; + const double xi_268 = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0]; + const double xi_269 = _data_pdfs_20_37_10[_stride_pdfs_0*ctr_0]; + const double xi_270 = _data_pdfs_20_315_10[_stride_pdfs_0*ctr_0]; + const double xi_271 = _data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_272 = _data_pdfs_20_314_10[_stride_pdfs_0*ctr_0]; + const double xi_273 = _data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_274 = _data_pdfs_20_39_10[_stride_pdfs_0*ctr_0]; + const double xi_275 = _data_pdfs_20_313_10[_stride_pdfs_0*ctr_0]; + const double xi_276 = _data_pdfs_20_317_10[_stride_pdfs_0*ctr_0]; + const double xi_277 = _data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_278 = _data_pdfs_20_32_10[_stride_pdfs_0*ctr_0]; + const double xi_279 = _data_pdfs_20_312_10[_stride_pdfs_0*ctr_0]; + const double xi_280 = _data_pdfs_20_316_10[_stride_pdfs_0*ctr_0]; + + double random_7_0; + double random_7_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 7, seed, random_7_0, random_7_1); + + + double random_6_0; + double random_6_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 6, seed, random_6_0, random_6_1); + + + double random_5_0; + double random_5_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 5, seed, random_5_0, random_5_1); + + + double random_4_0; + double random_4_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 4, seed, random_4_0, random_4_1); + + + double random_3_0; + double random_3_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1); + + + double random_2_0; + double random_2_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1); + + + double random_1_0; + double random_1_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1); + + + double random_0_0; + double random_0_1; + philox_double2(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1); + + const double xi_0 = xi_259 + xi_265; + const double xi_1 = xi_262 + xi_272; + const double xi_2 = xi_0 + xi_1 + xi_266; + const double xi_3 = xi_264 + xi_270; + const double xi_4 = xi_260 + xi_3; + const double xi_5 = xi_269 + xi_4; + const double xi_6 = xi_261 + xi_279; + const double xi_7 = xi_275 + xi_6; + const double xi_8 = xi_268 + xi_274; + const double xi_9 = xi_278 + xi_280; + const double xi_10 = xi_267 + xi_276; + const double xi_12 = -xi_276; + const double xi_13 = -xi_268; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -xi_275; + const double xi_16 = -xi_269; + const double xi_17 = -xi_274; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -xi_265; + const double xi_21 = xi_20 + xi_259; + const double xi_22 = -xi_279; + const double xi_23 = -xi_278; + const double xi_24 = -xi_280; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -xi_262; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -xi_270; + const double xi_29 = -xi_267; + const double xi_30 = xi_24 + xi_264 + xi_28 + xi_29; + const double xi_31 = xi_275 + xi_276; + const double xi_32 = xi_269 + xi_31 + xi_8; + const double xi_33 = xi_16 + xi_21 + xi_274; + const double xi_34 = xi_26 + xi_276; + const double xi_35 = xi_15 + xi_272; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_274 + xi_279 + xi_9; + const double xi_38 = xi_28 + xi_280; + const double xi_39 = xi_22 + xi_264 + xi_38; + const double xi_57 = xi_271*0.166666666666667; + const double xi_65 = xi_277*0.166666666666667; + const double xi_69 = xi_273*0.166666666666667; + const double xi_72 = xi_271*0.5; + const double xi_76 = xi_277*0.0833333333333333; + const double xi_80 = xi_271*0.0833333333333333; + const double xi_90 = xi_273*0.0833333333333333; + const double xi_108 = -xi_263; + const double xi_109 = xi_108 + xi_261*3.0 + xi_267*3.0; + const double xi_110 = omega_even*(xi_109 + xi_260*3.0 + xi_264*-3.0 + xi_270*-3.0 + xi_278*3.0 + xi_279*-3.0 + xi_280*-3.0); + const double xi_111 = xi_264*2.0 + xi_270*2.0 + xi_279*2.0 + xi_280*2.0; + const double xi_112 = xi_111 + xi_266*5.0 + xi_268*5.0; + const double xi_113 = omega_even*(xi_109 + xi_112 + xi_260*-2.0 + xi_262*-5.0 + xi_272*-5.0 + xi_275*-5.0 + xi_276*-5.0 + xi_278*-2.0); + const double xi_116 = -xi_264; + const double xi_117 = xi_116 + xi_22; + const double xi_118 = -xi_259; + const double xi_121 = -xi_272; + const double xi_122 = xi_121 + xi_19 + xi_27; + const double xi_124 = xi_275*2.0; + const double xi_125 = xi_272*2.0; + const double xi_126 = xi_262*2.0 + xi_276*2.0; + const double xi_127 = omega_even*(xi_108 + xi_112 + xi_124 + xi_125 + xi_126 + xi_259*-7.0 + xi_260*5.0 + xi_261*-4.0 + xi_265*-7.0 + xi_267*-4.0 + xi_269*-7.0 + xi_274*-7.0 + xi_278*5.0); + const double xi_128 = xi_116 + xi_23 + xi_260 + xi_279 + xi_38; + const double xi_130 = xi_128*xi_129; + const double xi_135 = random_5_1 - 0.5; + const double xi_140 = xi_269*2.0; + const double xi_141 = xi_265*2.0; + const double xi_142 = xi_259*-2.0 + xi_274*2.0; + const double xi_143 = -xi_140 + xi_141 + xi_142 + xi_25 + xi_4; + const double xi_145 = xi_143*xi_144; + const double xi_146 = random_3_0 - 0.5; + const double xi_151 = random_0_1 - 0.5; + const double xi_168 = xi_121 + xi_13 + xi_266 + xi_275 + xi_34; + const double xi_169 = xi_129*xi_168; + const double xi_170 = random_4_1 - 0.5; + const double xi_172 = xi_14 + xi_140 - xi_141 + xi_142 + xi_262 + xi_266 + xi_35; + const double xi_173 = xi_144*xi_172; + const double xi_174 = random_4_0 - 0.5; + const double xi_179 = xi_270 + xi_280; + const double xi_180 = xi_117 + xi_179 + xi_261 + xi_29; + const double xi_181 = xi_129*xi_180; + const double xi_184 = random_5_0 - 0.5; + const double xi_186 = -xi_124 - xi_125 + xi_126 + xi_30 + xi_6; + const double xi_187 = xi_144*xi_186; + const double xi_188 = random_3_1 - 0.5; + const double xi_195 = xi_127*0.0138888888888889; + const double xi_216 = xi_113*-0.00714285714285714; + const double xi_218 = xi_110*0.025; + const double xi_223 = xi_186*xi_222; + const double xi_225 = xi_180*xi_224; + const double xi_234 = xi_143*xi_222; + const double xi_235 = xi_128*xi_224; + const double xi_243 = xi_113*0.0178571428571429; + const double xi_249 = xi_168*xi_224; + const double xi_250 = xi_172*xi_222; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_263 + xi_8 + xi_9; + const double xi_11 = 1 / (rho); + const double xi_101 = kT*rho; + const double xi_102 = sqrt(xi_101*(-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0)); + const double xi_103 = xi_102*(random_6_0 - 0.5)*3.7416573867739413; + const double xi_104 = xi_102*(random_7_0 - 0.5)*5.4772255750516612; + const double xi_106 = xi_105*sqrt(xi_101*(-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0))*(random_2_1 - 0.5); + const double xi_107 = xi_102*(random_6_1 - 0.5)*8.3666002653407556; + const double xi_136 = sqrt(xi_101*(-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0)); + const double xi_137 = xi_136*1.4142135623730951; + const double xi_138 = xi_137*0.5; + const double xi_139 = xi_135*xi_138; + const double xi_147 = xi_105*xi_136; + const double xi_148 = xi_147*0.166666666666667; + const double xi_149 = xi_146*xi_148; + const double xi_150 = -xi_145 - xi_149; + const double xi_152 = sqrt(xi_101*(-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0)); + const double xi_153 = xi_152*0.5; + const double xi_154 = xi_151*xi_153; + const double xi_158 = xi_103*-0.119047619047619 + xi_127*-0.0198412698412698; + const double xi_160 = xi_152*(random_0_0 - 0.5)*1.7320508075688772; + const double xi_164 = xi_145 + xi_149; + const double xi_171 = xi_138*xi_170; + const double xi_175 = xi_148*xi_174; + const double xi_176 = xi_173 + xi_175; + const double xi_178 = -xi_173 - xi_175; + const double xi_185 = xi_138*xi_184; + const double xi_189 = xi_148*xi_188; + const double xi_190 = -xi_187 - xi_189; + const double xi_192 = xi_187 + xi_189; + const double xi_193 = xi_151*xi_152*0.25; + const double xi_196 = xi_103*0.0833333333333333; + const double xi_206 = xi_153*(random_1_0 - 0.5); + const double xi_215 = xi_153*(random_2_0 - 0.5); + const double xi_219 = xi_107*-0.0142857142857143; + const double xi_220 = xi_104*0.05; + const double xi_226 = xi_147*0.0833333333333333; + const double xi_227 = xi_188*xi_226; + const double xi_228 = xi_137*0.25; + const double xi_229 = xi_184*xi_228; + const double xi_231 = xi_103*-0.0238095238095238 + xi_127*-0.00396825396825397; + const double xi_236 = xi_146*xi_226; + const double xi_237 = xi_135*xi_228; + const double xi_241 = -xi_193; + const double xi_244 = xi_107*0.0357142857142857; + const double xi_246 = xi_153*(random_1_1 - 0.5); + const double xi_251 = xi_170*xi_228; + const double xi_252 = xi_174*xi_226; + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_41 = u_0*xi_277; + const double xi_42 = xi_41*0.333333333333333; + const double xi_48 = -xi_42; + const double xi_114 = rho*(u_0*u_0); + const double xi_165 = rho*u_0; + const double xi_166 = -vel0Term + xi_165 + xi_32; + const double xi_167 = xi_133*xi_166; + const double xi_202 = xi_166*xi_197; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_43 = u_1*xi_271; + const double xi_44 = xi_43*0.333333333333333; + const double xi_49 = -xi_44; + const double xi_71 = u_1*0.5; + const double xi_74 = xi_73*(u_0*xi_72 + xi_277*xi_71); + const double xi_75 = -xi_74; + const double xi_119 = rho*(u_1*u_1); + const double xi_120 = xi_118 + xi_119 + xi_20; + const double xi_131 = rho*u_1; + const double xi_132 = -vel1Term + xi_118 + xi_131 + xi_265 + xi_37; + const double xi_134 = xi_132*xi_133; + const double xi_198 = xi_132*xi_197; + const double u_2 = xi_11*(vel2Term + xi_27 + xi_272 + xi_30); + const double xi_45 = u_2*xi_273; + const double xi_46 = xi_45*0.333333333333333; + const double xi_47 = (xi_40 + 2.0)*(xi_42 + xi_44 + xi_46); + const double xi_50 = xi_45*0.666666666666667 + xi_48 + xi_49; + const double xi_54 = -xi_46; + const double xi_55 = xi_43*0.666666666666667 + xi_48 + xi_54; + const double xi_56 = xi_41*0.666666666666667 + xi_49 + xi_54; + const double xi_59 = xi_50*xi_58; + const double xi_60 = -xi_59; + const double xi_61 = xi_56*xi_58; + const double xi_62 = -xi_61; + const double xi_64 = xi_55*xi_63 + xi_60 + xi_62; + const double xi_66 = xi_55*xi_58; + const double xi_67 = -xi_66; + const double xi_68 = xi_56*xi_63 + xi_60 + xi_67; + const double xi_70 = xi_50*xi_63 + xi_62 + xi_67; + const double xi_77 = xi_61 - xi_76; + const double xi_79 = -xi_50*xi_78; + const double xi_81 = xi_47*0.125; + const double xi_82 = xi_66 + xi_81; + const double xi_83 = xi_80 + xi_82; + const double xi_84 = xi_79 + xi_83; + const double xi_85 = xi_61 + xi_76; + const double xi_86 = -xi_80 + xi_82; + const double xi_87 = xi_79 + xi_86; + const double xi_88 = xi_73*(u_2*xi_72 + xi_273*xi_71); + const double xi_89 = -xi_56*xi_78; + const double xi_91 = xi_59 + xi_90; + const double xi_92 = xi_89 + xi_91; + const double xi_93 = -xi_88; + const double xi_94 = xi_73*(u_0*xi_273*0.5 + u_2*xi_277*0.5); + const double xi_95 = -xi_94; + const double xi_96 = -xi_55*xi_78; + const double xi_97 = xi_81 + xi_91 + xi_96; + const double xi_98 = xi_59 - xi_90; + const double xi_99 = xi_89 + xi_98; + const double xi_100 = xi_81 + xi_96 + xi_98; + const double xi_115 = rho*(u_2*u_2); + const double xi_123 = omega_bulk*(xi_114 + xi_115 + xi_117 + xi_120 + xi_122 + xi_24 + xi_263 + xi_28); + const double xi_155 = -xi_115 + xi_261 + xi_267; + const double xi_156 = omega_shear*(xi_1 + xi_120 + xi_155 + xi_18 + xi_23 - xi_260 + xi_31); + const double xi_157 = xi_156*0.125; + const double xi_159 = omega_shear*(xi_111 + xi_114*2.0 + xi_118 - xi_119 + xi_122 + xi_155 + xi_20 + xi_260 + xi_266*-2.0 + xi_268*-2.0 + xi_278); + const double xi_161 = xi_159*-0.0416666666666667 + xi_160*-0.166666666666667; + const double xi_162 = xi_104*-0.1 + xi_110*-0.05 + xi_161; + const double xi_163 = xi_107*0.0285714285714286 + xi_113*0.0142857142857143 + xi_154 + xi_157 + xi_158 + xi_162; + const double xi_177 = xi_107*-0.0714285714285714 + xi_113*-0.0357142857142857 + xi_158 + xi_159*0.0833333333333333 + xi_160*0.333333333333333; + const double xi_182 = rho*u_2 - vel2Term + xi_10 + xi_116 + xi_121 + xi_179 + xi_262; + const double xi_183 = xi_133*xi_182; + const double xi_191 = xi_103*0.0952380952380952 + xi_107*-0.0428571428571429 + xi_113*-0.0214285714285714 + xi_127*0.0158730158730159 - xi_154 - xi_157 + xi_162; + const double xi_194 = xi_156*0.0625; + const double xi_199 = xi_106*0.0833333333333333 + xi_123*0.0416666666666667; + const double xi_200 = xi_198 + xi_199; + const double xi_201 = xi_164 + xi_193 + xi_194 + xi_195 + xi_196 + xi_200; + const double xi_203 = xi_159*0.0208333333333333 + xi_160*0.0833333333333333; + const double xi_204 = -xi_202 + xi_203; + const double xi_205 = xi_178 + xi_204; + const double xi_211 = xi_202 + xi_203; + const double xi_212 = xi_176 + xi_211; + const double xi_213 = -xi_198 + xi_199; + const double xi_214 = xi_150 + xi_193 + xi_194 + xi_195 + xi_196 + xi_213; + const double xi_230 = xi_182*xi_197; + const double xi_232 = xi_230 + xi_231; + const double xi_233 = -xi_223 + xi_225 - xi_227 + xi_229 + xi_232; + const double xi_238 = xi_200 - xi_234 + xi_235 - xi_236 + xi_237; + const double xi_239 = xi_213 + xi_234 - xi_235 + xi_236 - xi_237; + const double xi_242 = -xi_194; + const double xi_245 = xi_192 + xi_199 + xi_232 + xi_241 + xi_242 + xi_243 + xi_244; + const double xi_253 = xi_204 - xi_249 + xi_250 - xi_251 + xi_252; + const double xi_255 = xi_211 + xi_249 - xi_250 + xi_251 - xi_252; + const double xi_256 = -xi_230 + xi_231; + const double xi_257 = xi_223 - xi_225 + xi_227 - xi_229 + xi_256; + const double xi_258 = xi_190 + xi_199 + xi_241 + xi_242 + xi_243 + xi_244 + xi_256; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_208 = xi_207*(-p_1 + u_0*xi_131); + const double xi_209 = -xi_206 - xi_208; + const double xi_210 = xi_206 + xi_208; + const double p_2 = xi_36; + const double xi_247 = xi_207*(-p_2 + u_2*xi_165); + const double xi_248 = -xi_246 - xi_247; + const double xi_254 = xi_246 + xi_247; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_217 = xi_207*(-p_5 + u_2*xi_131); + const double xi_221 = xi_161 + xi_215 + xi_216 + xi_217 + xi_218 + xi_219 + xi_220; + const double xi_240 = xi_161 - xi_215 + xi_216 - xi_217 + xi_218 + xi_219 + xi_220; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_280 + xi_3 + xi_7; + const double forceTerm_0 = xi_47*-1.5 - xi_50*xi_53 - xi_53*xi_55 - xi_53*xi_56; + const double forceTerm_1 = xi_57 + xi_64; + const double forceTerm_2 = -xi_57 + xi_64; + const double forceTerm_3 = -xi_65 + xi_68; + const double forceTerm_4 = xi_65 + xi_68; + const double forceTerm_5 = xi_69 + xi_70; + const double forceTerm_6 = -xi_69 + xi_70; + const double forceTerm_7 = xi_75 + xi_77 + xi_84; + const double forceTerm_8 = xi_74 + xi_84 + xi_85; + const double forceTerm_9 = xi_74 + xi_77 + xi_87; + const double forceTerm_10 = xi_75 + xi_85 + xi_87; + const double forceTerm_11 = xi_83 + xi_88 + xi_92; + const double forceTerm_12 = xi_86 + xi_92 + xi_93; + const double forceTerm_13 = xi_77 + xi_95 + xi_97; + const double forceTerm_14 = xi_85 + xi_94 + xi_97; + const double forceTerm_15 = xi_83 + xi_93 + xi_99; + const double forceTerm_16 = xi_86 + xi_88 + xi_99; + const double forceTerm_17 = xi_100 + xi_77 + xi_94; + const double forceTerm_18 = xi_100 + xi_85 + xi_95; + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0] = forceTerm_0 + xi_103*0.142857142857143 + xi_104*0.2 - xi_106 + xi_107*0.0857142857142857 + xi_110*0.1 + xi_113*0.0428571428571429 + xi_123*-0.5 + xi_127*0.0238095238095238 + xi_263; + _data_pdfs_20_31_10[_stride_pdfs_0*ctr_0] = forceTerm_1 - xi_130 + xi_134 - xi_139 + xi_150 + xi_163 + xi_260; + _data_pdfs_20_32_10[_stride_pdfs_0*ctr_0] = forceTerm_2 + xi_130 - xi_134 + xi_139 + xi_163 + xi_164 + xi_278; + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0] = forceTerm_3 - xi_167 + xi_169 + xi_171 + xi_176 + xi_177 + xi_268; + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0] = forceTerm_4 + xi_167 - xi_169 - xi_171 + xi_177 + xi_178 + xi_266; + _data_pdfs_20_35_10[_stride_pdfs_0*ctr_0] = forceTerm_5 - xi_181 + xi_183 - xi_185 + xi_190 + xi_191 + xi_261; + _data_pdfs_20_36_10[_stride_pdfs_0*ctr_0] = forceTerm_6 + xi_181 - xi_183 + xi_185 + xi_191 + xi_192 + xi_267; + _data_pdfs_20_37_10[_stride_pdfs_0*ctr_0] = forceTerm_7 + xi_201 + xi_205 + xi_209 + xi_269; + _data_pdfs_20_38_10[_stride_pdfs_0*ctr_0] = forceTerm_8 + xi_201 + xi_210 + xi_212 + xi_259; + _data_pdfs_20_39_10[_stride_pdfs_0*ctr_0] = forceTerm_9 + xi_205 + xi_210 + xi_214 + xi_274; + _data_pdfs_20_310_10[_stride_pdfs_0*ctr_0] = forceTerm_10 + xi_209 + xi_212 + xi_214 + xi_265; + _data_pdfs_20_311_10[_stride_pdfs_0*ctr_0] = forceTerm_11 + xi_221 + xi_233 + xi_238 + xi_264; + _data_pdfs_20_312_10[_stride_pdfs_0*ctr_0] = forceTerm_12 + xi_233 + xi_239 + xi_240 + xi_279; + _data_pdfs_20_313_10[_stride_pdfs_0*ctr_0] = forceTerm_13 + xi_245 + xi_248 + xi_253 + xi_275; + _data_pdfs_20_314_10[_stride_pdfs_0*ctr_0] = forceTerm_14 + xi_245 + xi_254 + xi_255 + xi_272; + _data_pdfs_20_315_10[_stride_pdfs_0*ctr_0] = forceTerm_15 + xi_238 + xi_240 + xi_257 + xi_270; + _data_pdfs_20_316_10[_stride_pdfs_0*ctr_0] = forceTerm_16 + xi_221 + xi_239 + xi_257 + xi_280; + _data_pdfs_20_317_10[_stride_pdfs_0*ctr_0] = forceTerm_17 + xi_253 + xi_254 + xi_258 + xi_276; + _data_pdfs_20_318_10[_stride_pdfs_0*ctr_0] = forceTerm_18 + xi_248 + xi_255 + xi_258 + xi_262; + } } - } - } + } +} } -} // namespace internal_kernel_collide namespace internal_kernel_stream { -static FUNC_PREFIX void kernel_stream( - double *RESTRICT const _data_pdfs, double *RESTRICT _data_pdfs_tmp, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, - int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3) { - for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - for (int64_t ctr_0 = 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) { - _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void kernel_stream(double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3) +{ + for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + for (int64_t ctr_0 = 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) + { + _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } } -} // namespace internal_kernel_stream +} + -const real_t FluctuatingMRTLatticeModel::w[19] = { - 0.333333333333333, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778}; -const real_t FluctuatingMRTLatticeModel::wInv[19] = { - 3.00000000000000, 18.0000000000000, 18.0000000000000, 18.0000000000000, - 18.0000000000000, 18.0000000000000, 18.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000}; +const real_t FluctuatingMRTLatticeModel::w[19] = { 0.333333333333333,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778 }; +const real_t FluctuatingMRTLatticeModel::wInv[19] = { 3.00000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000 }; -void FluctuatingMRTLatticeModel::Sweep::streamCollide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); +void FluctuatingMRTLatticeModel::Sweep::streamCollide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - auto &lm = dynamic_cast *>(pdfs) - ->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &omega_odd = lm.omega_odd_; - auto &time_step = lm.time_step_; - auto &block_offset_1 = lm.block_offset_1_; - auto &force = lm.force_; - auto &block_offset_0 = lm.block_offset_0_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &kT = lm.kT_; - auto &seed = lm.seed_; - auto &block_offset_2 = lm.block_offset_2_; - auto &omega_bulk = lm.omega_bulk_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_force_0 = int64_t(force->xStride()); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_streamCollide::kernel_streamCollide( - _data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, - _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, - _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, - _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, block_offset_0, block_offset_1, - block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, - time_step); - pdfs->swapDataPointers(pdfs_tmp); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & force = lm.force_; + auto & omega_even = lm.omega_even_; + auto & omega_odd = lm.omega_odd_; + auto & block_offset_2 = lm.block_offset_2_; + auto & block_offset_0 = lm.block_offset_0_; + auto & omega_shear = lm.omega_shear_; + auto & block_offset_1 = lm.block_offset_1_; + auto & omega_bulk = lm.omega_bulk_; + auto & seed = lm.seed_; + auto & kT = lm.kT_; + auto & time_step = lm.time_step_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_force_0 = int64_t(force->xStride()); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_streamCollide::kernel_streamCollide(_data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, time_step); + pdfs->swapDataPointers(pdfs_tmp); + } -void FluctuatingMRTLatticeModel::Sweep::collide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); +void FluctuatingMRTLatticeModel::Sweep::collide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + - auto &lm = dynamic_cast *>(pdfs) - ->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); - auto &omega_odd = lm.omega_odd_; - auto &time_step = lm.time_step_; - auto &block_offset_1 = lm.block_offset_1_; - auto &force = lm.force_; - auto &block_offset_0 = lm.block_offset_0_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &kT = lm.kT_; - auto &seed = lm.seed_; - auto &block_offset_2 = lm.block_offset_2_; - auto &omega_bulk = lm.omega_bulk_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - const int64_t _stride_force_0 = int64_t(force->xStride()); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_kernel_collide::kernel_collide( - _data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, - _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, - block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, - omega_even, omega_odd, omega_shear, seed, time_step); + auto & force = lm.force_; + auto & omega_even = lm.omega_even_; + auto & omega_odd = lm.omega_odd_; + auto & block_offset_2 = lm.block_offset_2_; + auto & block_offset_0 = lm.block_offset_0_; + auto & omega_shear = lm.omega_shear_; + auto & block_offset_1 = lm.block_offset_1_; + auto & omega_bulk = lm.omega_bulk_; + auto & seed = lm.seed_; + auto & kT = lm.kT_; + auto & time_step = lm.time_step_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + const int64_t _stride_force_0 = int64_t(force->xStride()); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_kernel_collide::kernel_collide(_data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, time_step); } -void FluctuatingMRTLatticeModel::Sweep::stream( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); + +void FluctuatingMRTLatticeModel::Sweep::stream( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_0 = - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_1 = - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_2 = - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_stream::kernel_stream( - _data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, - _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, - _stride_pdfs_tmp_3); - pdfs->swapDataPointers(pdfs_tmp); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_stream::kernel_stream(_data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); + + pdfs->swapDataPointers(pdfs_tmp); + } + } // namespace lbm } // namespace walberla + + + // Buffer Packing namespace walberla { namespace mpi { -mpi::SendBuffer & -operator<<(mpi::SendBuffer &buf, - const ::walberla::lbm::FluctuatingMRTLatticeModel &lm) { - buf << lm.currentLevel; - return buf; +mpi::SendBuffer & operator<< (mpi::SendBuffer & buf, const ::walberla::lbm::FluctuatingMRTLatticeModel & lm) +{ + buf << lm.currentLevel; + return buf; } -mpi::RecvBuffer &operator>>(mpi::RecvBuffer &buf, - ::walberla::lbm::FluctuatingMRTLatticeModel &lm) { - buf >> lm.currentLevel; - return buf; +mpi::RecvBuffer & operator>> (mpi::RecvBuffer & buf, ::walberla::lbm::FluctuatingMRTLatticeModel & lm) +{ + buf >> lm.currentLevel; + return buf; } + } // namespace mpi } // namespace walberla -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic pop +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic pop #endif \ No newline at end of file diff --git a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.h b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.h index 8b510541764..ab6ab7be70f 100644 --- a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.h +++ b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.h @@ -696,16 +696,101 @@ template<> struct PressureTensor { template< typename FieldPtrOrIterator > - static void get( Matrix3< real_t > & /* pressureTensor */, const FluctuatingMRTLatticeModel & /* latticeModel */, const FieldPtrOrIterator & /* it */ ) + static void get( Matrix3< real_t > & pressureTensor, const FluctuatingMRTLatticeModel & lm, const FieldPtrOrIterator & it ) { - WALBERLA_ABORT("Not implemented"); + const auto x = it.x(); + const auto y = it.y(); + const auto z = it.z(); + + const real_t f_0 = it[0]; + const real_t f_1 = it[1]; + const real_t f_2 = it[2]; + const real_t f_3 = it[3]; + const real_t f_4 = it[4]; + const real_t f_5 = it[5]; + const real_t f_6 = it[6]; + const real_t f_7 = it[7]; + const real_t f_8 = it[8]; + const real_t f_9 = it[9]; + const real_t f_10 = it[10]; + const real_t f_11 = it[11]; + const real_t f_12 = it[12]; + const real_t f_13 = it[13]; + const real_t f_14 = it[14]; + const real_t f_15 = it[15]; + const real_t f_16 = it[16]; + const real_t f_17 = it[17]; + const real_t f_18 = it[18]; + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } template< typename PdfField_T > - static void get( Matrix3< real_t > & /* pressureTensor */, const FluctuatingMRTLatticeModel & /* latticeModel */, const PdfField_T & /* pdf */, - const cell_idx_t /* x */, const cell_idx_t /* y */, const cell_idx_t /* z */ ) + static void get( Matrix3< real_t > & pressureTensor , const FluctuatingMRTLatticeModel & lm , const PdfField_T & pdf, + const cell_idx_t x, const cell_idx_t y, const cell_idx_t z ) { - WALBERLA_ABORT("Not implemented"); + const real_t & xyz0 = pdf(x,y,z,0); + const real_t f_0 = pdf.getF( &xyz0, 0); + const real_t f_1 = pdf.getF( &xyz0, 1); + const real_t f_2 = pdf.getF( &xyz0, 2); + const real_t f_3 = pdf.getF( &xyz0, 3); + const real_t f_4 = pdf.getF( &xyz0, 4); + const real_t f_5 = pdf.getF( &xyz0, 5); + const real_t f_6 = pdf.getF( &xyz0, 6); + const real_t f_7 = pdf.getF( &xyz0, 7); + const real_t f_8 = pdf.getF( &xyz0, 8); + const real_t f_9 = pdf.getF( &xyz0, 9); + const real_t f_10 = pdf.getF( &xyz0, 10); + const real_t f_11 = pdf.getF( &xyz0, 11); + const real_t f_12 = pdf.getF( &xyz0, 12); + const real_t f_13 = pdf.getF( &xyz0, 13); + const real_t f_14 = pdf.getF( &xyz0, 14); + const real_t f_15 = pdf.getF( &xyz0, 15); + const real_t f_16 = pdf.getF( &xyz0, 16); + const real_t f_17 = pdf.getF( &xyz0, 17); + const real_t f_18 = pdf.getF( &xyz0, 18); + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } }; diff --git a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.cpp b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.cpp index 806b74fdf1d..827b84010f1 100644 --- a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.cpp +++ b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.cpp @@ -18,3751 +18,1204 @@ #include -#include "FluctuatingMRTLatticeModelAvx.h" #include "core/DataTypes.h" #include "core/Macros.h" #include "lbm/field/PdfField.h" #include "lbm/sweeps/Streaming.h" +#include "FluctuatingMRTLatticeModelAvx.h" #ifdef _MSC_VER -#pragma warning(disable : 4458) +# pragma warning( disable : 4458 ) #endif #define FUNC_PREFIX -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#pragma GCC diagnostic ignored "-Wshadow" -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-parameter" +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wfloat-equal" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Wunused-parameter" #endif + #include "philox_rand.h" #include + + using namespace std; namespace walberla { namespace lbm { namespace internal_kernel_streamCollide { -static FUNC_PREFIX void kernel_streamCollide( - double *RESTRICT const _data_force, double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, - int64_t const _size_force_1, int64_t const _size_force_2, - int64_t const _stride_force_1, int64_t const _stride_force_2, - int64_t const _stride_force_3, int64_t const _stride_pdfs_1, - int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, - int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3, uint32_t block_offset_0, - uint32_t block_offset_1, uint32_t block_offset_2, double kT, - double omega_bulk, double omega_even, double omega_odd, double omega_shear, - uint32_t seed, uint32_t time_step) { - const double xi_25 = -omega_bulk; - const double xi_36 = -omega_shear; - const double xi_37 = xi_36 + 2.0; - const double xi_38 = xi_37 * 0.5; - const double xi_43 = xi_37 * 0.0833333333333333; - const double xi_48 = xi_37 * 0.166666666666667; - const double xi_58 = xi_37 * 0.25; - const double xi_63 = xi_37 * 0.0416666666666667; - const double xi_90 = 2.4494897427831779; - const double xi_115 = omega_odd * 0.25; - const double xi_131 = omega_odd * 0.0833333333333333; - const double xi_196 = omega_shear * 0.25; - const double xi_211 = omega_odd * 0.0416666666666667; - const double xi_213 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_120 = rr_0 * 0.166666666666667; - const double xi_186 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - for (int64_t ctr_0 = 1; - ctr_0 < ((_size_force_0 - 2) % (4) == 0 - ? _size_force_0 - 2 - : ((int64_t)((_size_force_0 - 2) / (4)) + 1) * (4)) + - 1; - ctr_0 += 4) { - - __m256d random_7_0; - __m256d random_7_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 7, seed, - random_7_0, random_7_1); - - __m256d random_6_0; - __m256d random_6_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 6, seed, - random_6_0, random_6_1); - - __m256d random_5_0; - __m256d random_5_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 5, seed, - random_5_0, random_5_1); - - __m256d random_4_0; - __m256d random_4_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 4, seed, - random_4_0, random_4_1); - - __m256d random_3_0; - __m256d random_3_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, - random_3_0, random_3_1); - - __m256d random_2_0; - __m256d random_2_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, - random_2_0, random_2_1); - - __m256d random_1_0; - __m256d random_1_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, - random_1_0, random_1_1); - - __m256d random_0_0; - __m256d random_0_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, - random_0_0, random_0_1); - - const __m256d xi_0 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1]), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - const __m256d xi_1 = _mm256_add_pd( - xi_0, _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1])); - const __m256d xi_2 = _mm256_add_pd( - _mm256_add_pd(_mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0]), - _mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0])), - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - const __m256d xi_3 = - _mm256_add_pd(_mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0]), - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0])); - const __m256d xi_4 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1]), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])); - const __m256d xi_5 = - _mm256_add_pd(_mm256_load_pd(&_data_pdfs_20_32_11[ctr_0]), - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_6 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1]), - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0])); - const __m256d xi_8 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])); - const __m256d xi_9 = _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])), - xi_8); - const __m256d xi_10 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])); - const __m256d xi_11 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_12 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1])); - const __m256d xi_13 = _mm256_add_pd(_mm256_add_pd(xi_10, xi_11), xi_12); - const __m256d xi_14 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0])); - const __m256d xi_15 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - const __m256d xi_16 = _mm256_add_pd(xi_14, xi_15); - const __m256d xi_17 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_18 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0])); - const __m256d xi_19 = _mm256_add_pd(xi_17, xi_18); - const __m256d xi_20 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])); - const __m256d xi_21 = _mm256_add_pd(xi_10, xi_20); - const __m256d xi_22 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0])); - const __m256d xi_23 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0])); - const __m256d xi_24 = - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_17, xi_22), xi_23), - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - const __m256d xi_42 = - _mm256_mul_pd(_mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667), - _mm256_load_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_50 = - _mm256_mul_pd(_mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667), - _mm256_load_pd(&_data_force_20_30_10[ctr_0])); - const __m256d xi_54 = - _mm256_mul_pd(_mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667), - _mm256_load_pd(&_data_force_20_32_10[ctr_0])); - const __m256d xi_57 = - _mm256_mul_pd(_mm256_set_pd(0.5, 0.5, 0.5, 0.5), - _mm256_load_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_61 = - _mm256_mul_pd(_mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333), - _mm256_load_pd(&_data_force_20_30_10[ctr_0])); - const __m256d xi_65 = - _mm256_mul_pd(_mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333), - _mm256_load_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_75 = - _mm256_mul_pd(_mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333), - _mm256_load_pd(&_data_force_20_32_10[ctr_0])); - const __m256d xi_93 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0])); - const __m256d xi_94 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0])), - _mm256_mul_pd(_mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0]))), - xi_93); - const __m256d xi_95 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_load_pd( - &_data_pdfs_21_315_1m1[ctr_0])), - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_load_pd( - &_data_pdfs_21_316_11[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_load_pd( - &_data_pdfs_2m1_311_1m1[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0]))), - _mm256_mul_pd(_mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0]))), - xi_94), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_96 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_load_pd( - &_data_pdfs_21_315_1m1[ctr_0])), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_load_pd( - &_data_pdfs_21_316_11[ctr_0]))), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0]))), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0]))); - const __m256d xi_97 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1])), - _mm256_mul_pd( - _mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1]))), - xi_96); - const __m256d xi_98 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, - -5.0), - _mm256_loadu_pd( - &_data_pdfs_21_317_10[ctr_0 + - 1])), - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, - -5.0), - _mm256_loadu_pd( - &_data_pdfs_21_318_10[ctr_0 - - 1]))), - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, -5.0), - _mm256_loadu_pd( - &_data_pdfs_2m1_313_10[ctr_0 + - 1]))), - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, -5.0), - _mm256_loadu_pd( - &_data_pdfs_2m1_314_10[ctr_0 - 1]))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0), - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0), - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0]))), - xi_94), - xi_97), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_101 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - const __m256d xi_102 = _mm256_add_pd(xi_101, xi_18); - const __m256d xi_103 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])); - const __m256d xi_106 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - const __m256d xi_107 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_106, xi_11), xi_15), xi_21); - const __m256d xi_109 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_110 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - const __m256d xi_111 = _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1]))); - const __m256d xi_112 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd( - -7.0, -7.0, - -7.0, -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_310_11 - [ctr_0 - - 1])), - _mm256_mul_pd( - _mm256_set_pd( - -7.0, -7.0, - -7.0, -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_37_1m1 - [ctr_0 + - 1]))), - _mm256_mul_pd( - _mm256_set_pd( - -7.0, -7.0, - -7.0, -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_38_1m1 - [ctr_0 - - 1]))), - _mm256_mul_pd( - _mm256_set_pd( - -7.0, -7.0, -7.0, - -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_39_11 - [ctr_0 + 1]))), - _mm256_mul_pd( - _mm256_set_pd(-4.0, -4.0, - -4.0, -4.0), - _mm256_load_pd( - &_data_pdfs_21_36_10 - [ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-4.0, -4.0, -4.0, - -4.0), - _mm256_load_pd( - &_data_pdfs_2m1_35_10 - [ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_load_pd( - &_data_pdfs_20_31_1m1[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_load_pd( - &_data_pdfs_20_32_11[ctr_0]))), - xi_109), - xi_110), - xi_111), - xi_93), - xi_97), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_113 = _mm256_add_pd( - xi_101, _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0])); - const __m256d xi_114 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_113, xi_14), xi_22), - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0])), - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_116 = _mm256_mul_pd( - xi_114, _mm256_set_pd(xi_115, xi_115, xi_115, xi_115)); - const __m256d xi_118 = _mm256_add_pd( - xi_103, _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - const __m256d xi_122 = - _mm256_add_pd(random_5_1, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_127 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - const __m256d xi_128 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - const __m256d xi_129 = _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(-2.0, -2.0, -2.0, -2.0), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1]))); - const __m256d xi_130 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_127, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_128), - xi_129), - xi_14), - xi_19), - xi_2); - const __m256d xi_132 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_131, xi_131, xi_131, xi_131)); - const __m256d xi_133 = - _mm256_add_pd(random_3_0, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_138 = - _mm256_add_pd(random_0_1, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_142 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1]), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_156 = _mm256_add_pd( - xi_106, _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_157 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_12, xi_156), xi_20), - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1])), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])); - const __m256d xi_158 = _mm256_mul_pd( - xi_157, _mm256_set_pd(xi_115, xi_115, xi_115, xi_115)); - const __m256d xi_159 = - _mm256_add_pd(random_4_1, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_161 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_128, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_1), - xi_127), - xi_129), - xi_13); - const __m256d xi_162 = _mm256_mul_pd( - xi_161, _mm256_set_pd(xi_131, xi_131, xi_131, xi_131)); - const __m256d xi_163 = - _mm256_add_pd(random_4_0, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_168 = - _mm256_add_pd(_mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0]), - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_169 = - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_102, xi_168), xi_23), - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0])); - const __m256d xi_170 = _mm256_mul_pd( - xi_169, _mm256_set_pd(xi_115, xi_115, xi_115, xi_115)); - const __m256d xi_173 = - _mm256_add_pd(random_5_0, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_175 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_109, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_110, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_111), - xi_24), - xi_3); - const __m256d xi_176 = _mm256_mul_pd( - xi_175, _mm256_set_pd(xi_131, xi_131, xi_131, xi_131)); - const __m256d xi_177 = - _mm256_add_pd(random_3_1, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_184 = _mm256_mul_pd( - xi_112, _mm256_set_pd(0.0138888888888889, 0.0138888888888889, - 0.0138888888888889, 0.0138888888888889)); - const __m256d xi_205 = _mm256_mul_pd( - xi_98, _mm256_set_pd(-0.00714285714285714, -0.00714285714285714, - -0.00714285714285714, -0.00714285714285714)); - const __m256d xi_207 = - _mm256_mul_pd(xi_95, _mm256_set_pd(0.025, 0.025, 0.025, 0.025)); - const __m256d xi_212 = _mm256_mul_pd( - xi_175, _mm256_set_pd(xi_211, xi_211, xi_211, xi_211)); - const __m256d xi_214 = _mm256_mul_pd( - xi_169, _mm256_set_pd(xi_213, xi_213, xi_213, xi_213)); - const __m256d xi_223 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_211, xi_211, xi_211, xi_211)); - const __m256d xi_224 = _mm256_mul_pd( - xi_114, _mm256_set_pd(xi_213, xi_213, xi_213, xi_213)); - const __m256d xi_232 = _mm256_mul_pd( - xi_98, _mm256_set_pd(0.0178571428571429, 0.0178571428571429, - 0.0178571428571429, 0.0178571428571429)); - const __m256d xi_238 = _mm256_mul_pd( - xi_157, _mm256_set_pd(xi_213, xi_213, xi_213, xi_213)); - const __m256d xi_239 = _mm256_mul_pd( - xi_161, _mm256_set_pd(xi_211, xi_211, xi_211, xi_211)); - const __m256d vel0Term = _mm256_add_pd( - _mm256_add_pd(xi_1, - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])); - const __m256d vel1Term = _mm256_add_pd( - xi_2, _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - const __m256d vel2Term = _mm256_add_pd( - xi_3, _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d rho = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel0Term, vel1Term), - vel2Term), - xi_4), - xi_5), - xi_6), - _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0])); - const __m256d xi_7 = - _mm256_div_pd(_mm256_set_pd(1.0, 1.0, 1.0, 1.0), rho); - const __m256d xi_86 = _mm256_mul_pd(rho, _mm256_set_pd(kT, kT, kT, kT)); - const __m256d xi_87 = _mm256_sqrt_pd(_mm256_mul_pd( - xi_86, - _mm256_set_pd(-((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0, - -((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0, - -((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0, - -((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0))); - const __m256d xi_88 = _mm256_mul_pd( - _mm256_mul_pd(xi_87, - _mm256_add_pd(random_6_0, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - _mm256_set_pd(3.7416573867739413, 3.7416573867739413, - 3.7416573867739413, 3.7416573867739413)); - const __m256d xi_89 = _mm256_mul_pd( - _mm256_mul_pd(xi_87, - _mm256_add_pd(random_7_0, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - _mm256_set_pd(5.4772255750516612, 5.4772255750516612, - 5.4772255750516612, 5.4772255750516612)); - const __m256d xi_91 = _mm256_mul_pd( - _mm256_mul_pd( - _mm256_sqrt_pd(_mm256_mul_pd( - xi_86, - _mm256_set_pd(-((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0, - -((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0, - -((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0, - -((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0))), - _mm256_add_pd(random_2_1, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - _mm256_set_pd(xi_90, xi_90, xi_90, xi_90)); - const __m256d xi_92 = _mm256_mul_pd( - _mm256_mul_pd(xi_87, - _mm256_add_pd(random_6_1, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - _mm256_set_pd(8.3666002653407556, 8.3666002653407556, - 8.3666002653407556, 8.3666002653407556)); - const __m256d xi_123 = _mm256_sqrt_pd(_mm256_mul_pd( - xi_86, - _mm256_set_pd(-((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0, - -((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0, - -((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0, - -((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0))); - const __m256d xi_124 = _mm256_mul_pd( - xi_123, _mm256_set_pd(1.4142135623730951, 1.4142135623730951, - 1.4142135623730951, 1.4142135623730951)); - const __m256d xi_125 = - _mm256_mul_pd(xi_124, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_126 = _mm256_mul_pd(xi_122, xi_125); - const __m256d xi_134 = - _mm256_mul_pd(xi_123, _mm256_set_pd(xi_90, xi_90, xi_90, xi_90)); - const __m256d xi_135 = _mm256_mul_pd( - xi_134, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_136 = _mm256_mul_pd(xi_133, xi_135); - const __m256d xi_137 = _mm256_add_pd( - _mm256_mul_pd(xi_132, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_136, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))); - const __m256d xi_139 = _mm256_sqrt_pd(_mm256_mul_pd( - xi_86, _mm256_set_pd(-((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0, - -((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0, - -((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0, - -((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0))); - const __m256d xi_140 = - _mm256_mul_pd(xi_139, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_141 = _mm256_mul_pd(xi_138, xi_140); - const __m256d xi_146 = _mm256_add_pd( - _mm256_mul_pd(xi_112, _mm256_set_pd(-0.0198412698412698, - -0.0198412698412698, - -0.0198412698412698, - -0.0198412698412698)), - _mm256_mul_pd( - xi_88, _mm256_set_pd(-0.119047619047619, -0.119047619047619, - -0.119047619047619, -0.119047619047619))); - const __m256d xi_148 = _mm256_mul_pd( - _mm256_mul_pd(xi_139, - _mm256_add_pd(random_0_0, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - _mm256_set_pd(1.7320508075688772, 1.7320508075688772, - 1.7320508075688772, 1.7320508075688772)); - const __m256d xi_152 = _mm256_add_pd(xi_132, xi_136); - const __m256d xi_160 = _mm256_mul_pd(xi_125, xi_159); - const __m256d xi_164 = _mm256_mul_pd(xi_135, xi_163); - const __m256d xi_165 = _mm256_add_pd(xi_162, xi_164); - const __m256d xi_167 = _mm256_add_pd( - _mm256_mul_pd(xi_162, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_164, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))); - const __m256d xi_174 = _mm256_mul_pd(xi_125, xi_173); - const __m256d xi_178 = _mm256_mul_pd(xi_135, xi_177); - const __m256d xi_179 = _mm256_add_pd( - _mm256_mul_pd(xi_176, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_178, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))); - const __m256d xi_181 = _mm256_add_pd(xi_176, xi_178); - const __m256d xi_182 = - _mm256_mul_pd(_mm256_mul_pd(xi_138, xi_139), - _mm256_set_pd(0.25, 0.25, 0.25, 0.25)); - const __m256d xi_185 = _mm256_mul_pd( - xi_88, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_195 = _mm256_mul_pd( - xi_140, - _mm256_add_pd(random_1_0, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))); - const __m256d xi_204 = _mm256_mul_pd( - xi_140, - _mm256_add_pd(random_2_0, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))); - const __m256d xi_208 = _mm256_mul_pd( - xi_92, _mm256_set_pd(-0.0142857142857143, -0.0142857142857143, - -0.0142857142857143, -0.0142857142857143)); - const __m256d xi_209 = - _mm256_mul_pd(xi_89, _mm256_set_pd(0.05, 0.05, 0.05, 0.05)); - const __m256d xi_215 = _mm256_mul_pd( - xi_134, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_216 = _mm256_mul_pd(xi_177, xi_215); - const __m256d xi_217 = - _mm256_mul_pd(xi_124, _mm256_set_pd(0.25, 0.25, 0.25, 0.25)); - const __m256d xi_218 = _mm256_mul_pd(xi_173, xi_217); - const __m256d xi_220 = _mm256_add_pd( - _mm256_mul_pd(xi_112, _mm256_set_pd(-0.00396825396825397, - -0.00396825396825397, - -0.00396825396825397, - -0.00396825396825397)), - _mm256_mul_pd(xi_88, _mm256_set_pd(-0.0238095238095238, - -0.0238095238095238, - -0.0238095238095238, - -0.0238095238095238))); - const __m256d xi_225 = _mm256_mul_pd(xi_133, xi_215); - const __m256d xi_226 = _mm256_mul_pd(xi_122, xi_217); - const __m256d xi_230 = - _mm256_mul_pd(xi_182, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_233 = _mm256_mul_pd( - xi_92, _mm256_set_pd(0.0357142857142857, 0.0357142857142857, - 0.0357142857142857, 0.0357142857142857)); - const __m256d xi_235 = _mm256_mul_pd( - xi_140, - _mm256_add_pd(random_1_1, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))); - const __m256d xi_240 = _mm256_mul_pd(xi_159, xi_217); - const __m256d xi_241 = _mm256_mul_pd(xi_163, xi_215); - const __m256d u_0 = _mm256_mul_pd( - xi_7, _mm256_add_pd(_mm256_add_pd(vel0Term, xi_13), xi_9)); - const __m256d xi_26 = - _mm256_mul_pd(u_0, _mm256_load_pd(&_data_force_20_30_10[ctr_0])); - const __m256d xi_27 = _mm256_mul_pd( - xi_26, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_33 = - _mm256_mul_pd(xi_27, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_99 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_0, u_0))); - const __m256d xi_153 = _mm256_mul_pd(rho, u_0); - const __m256d xi_154 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(vel0Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_142), - xi_153), - xi_4), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - const __m256d xi_155 = _mm256_mul_pd( - xi_154, _mm256_set_pd(xi_120, xi_120, xi_120, xi_120)); - const __m256d xi_191 = _mm256_mul_pd( - xi_154, _mm256_set_pd(xi_186, xi_186, xi_186, xi_186)); - const __m256d u_1 = _mm256_mul_pd( - xi_7, - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel1Term, xi_16), xi_19), xi_8), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1]))); - const __m256d xi_28 = - _mm256_mul_pd(u_1, _mm256_load_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_29 = _mm256_mul_pd( - xi_28, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_34 = - _mm256_mul_pd(xi_29, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_56 = - _mm256_mul_pd(u_1, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_59 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_mul_pd(u_0, xi_57), - _mm256_mul_pd(xi_56, - _mm256_load_pd(&_data_force_20_30_10[ctr_0]))), - _mm256_set_pd(xi_58, xi_58, xi_58, xi_58)); - const __m256d xi_60 = - _mm256_mul_pd(xi_59, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_104 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_1, u_1))); - const __m256d xi_105 = - _mm256_add_pd(_mm256_add_pd(xi_103, xi_104), xi_9); - const __m256d xi_117 = _mm256_mul_pd(rho, u_1); - const __m256d xi_119 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(vel1Term, _mm256_set_pd(-1.0, -1.0, - -1.0, -1.0)), - xi_117), - xi_118), - xi_5), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])), - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0])); - const __m256d xi_121 = _mm256_mul_pd( - xi_119, _mm256_set_pd(xi_120, xi_120, xi_120, xi_120)); - const __m256d xi_187 = _mm256_mul_pd( - xi_119, _mm256_set_pd(xi_186, xi_186, xi_186, xi_186)); - const __m256d xi_197 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_0, xi_117), xi_118), - xi_8), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])), - _mm256_set_pd(xi_196, xi_196, xi_196, xi_196)); - const __m256d xi_198 = _mm256_add_pd( - _mm256_mul_pd(xi_195, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_197, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))); - const __m256d xi_199 = _mm256_add_pd(xi_195, xi_197); - const __m256d u_2 = _mm256_mul_pd( - xi_7, - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel2Term, xi_21), xi_24), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1]))); - const __m256d xi_30 = - _mm256_mul_pd(u_2, _mm256_load_pd(&_data_force_20_32_10[ctr_0])); - const __m256d xi_31 = _mm256_mul_pd( - xi_30, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_32 = _mm256_mul_pd( - _mm256_add_pd(_mm256_add_pd(xi_27, xi_29), xi_31), - _mm256_set_pd(xi_25 + 2.0, xi_25 + 2.0, xi_25 + 2.0, xi_25 + 2.0)); - const __m256d xi_35 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_30, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_33), - xi_34); - const __m256d xi_39 = - _mm256_mul_pd(xi_31, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_40 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_28, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_33), - xi_39); - const __m256d xi_41 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_26, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_34), - xi_39); - const __m256d xi_44 = - _mm256_mul_pd(xi_35, _mm256_set_pd(xi_43, xi_43, xi_43, xi_43)); - const __m256d xi_45 = - _mm256_mul_pd(xi_44, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_46 = - _mm256_mul_pd(xi_41, _mm256_set_pd(xi_43, xi_43, xi_43, xi_43)); - const __m256d xi_47 = - _mm256_mul_pd(xi_46, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_49 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_40, _mm256_set_pd(xi_48, xi_48, xi_48, xi_48)), - xi_45), - xi_47); - const __m256d xi_51 = - _mm256_mul_pd(xi_40, _mm256_set_pd(xi_43, xi_43, xi_43, xi_43)); - const __m256d xi_52 = - _mm256_mul_pd(xi_51, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_53 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_41, _mm256_set_pd(xi_48, xi_48, xi_48, xi_48)), - xi_45), - xi_52); - const __m256d xi_55 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_35, _mm256_set_pd(xi_48, xi_48, xi_48, xi_48)), - xi_47), - xi_52); - const __m256d xi_62 = _mm256_add_pd( - _mm256_mul_pd(xi_61, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_46); - const __m256d xi_64 = _mm256_mul_pd( - _mm256_mul_pd(xi_35, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_63, xi_63, xi_63, xi_63)); - const __m256d xi_66 = - _mm256_mul_pd(xi_32, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_67 = _mm256_add_pd(xi_51, xi_66); - const __m256d xi_68 = _mm256_add_pd(xi_65, xi_67); - const __m256d xi_69 = _mm256_add_pd(xi_64, xi_68); - const __m256d xi_70 = _mm256_add_pd(xi_46, xi_61); - const __m256d xi_71 = _mm256_add_pd( - _mm256_mul_pd(xi_65, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_67); - const __m256d xi_72 = _mm256_add_pd(xi_64, xi_71); - const __m256d xi_73 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_mul_pd(u_2, xi_57), - _mm256_mul_pd(xi_56, - _mm256_load_pd(&_data_force_20_32_10[ctr_0]))), - _mm256_set_pd(xi_58, xi_58, xi_58, xi_58)); - const __m256d xi_74 = _mm256_mul_pd( - _mm256_mul_pd(xi_41, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_63, xi_63, xi_63, xi_63)); - const __m256d xi_76 = _mm256_add_pd(xi_44, xi_75); - const __m256d xi_77 = _mm256_add_pd(xi_74, xi_76); - const __m256d xi_78 = - _mm256_mul_pd(xi_73, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_79 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_mul_pd(u_0, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)), - _mm256_load_pd(&_data_force_20_32_10[ctr_0])), - _mm256_mul_pd( - _mm256_mul_pd(u_2, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)), - _mm256_load_pd(&_data_force_20_30_10[ctr_0]))), - _mm256_set_pd(xi_58, xi_58, xi_58, xi_58)); - const __m256d xi_80 = - _mm256_mul_pd(xi_79, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_81 = _mm256_mul_pd( - _mm256_mul_pd(xi_40, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_63, xi_63, xi_63, xi_63)); - const __m256d xi_82 = _mm256_add_pd(_mm256_add_pd(xi_66, xi_76), xi_81); - const __m256d xi_83 = _mm256_add_pd( - _mm256_mul_pd(xi_75, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_44); - const __m256d xi_84 = _mm256_add_pd(xi_74, xi_83); - const __m256d xi_85 = _mm256_add_pd(_mm256_add_pd(xi_66, xi_81), xi_83); - const __m256d xi_100 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_2, u_2))); - const __m256d xi_108 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_100, xi_102), - xi_105), - xi_107), - xi_17), - xi_22), - xi_99), - _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0])), - _mm256_set_pd(omega_bulk, omega_bulk, omega_bulk, omega_bulk)); - const __m256d xi_143 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_100, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0])), - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0])); - const __m256d xi_144 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd( - &_data_pdfs_20_31_1m1[ctr_0])), - xi_0), - xi_105), - xi_142), - xi_143), - xi_16), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, omega_shear)); - const __m256d xi_145 = - _mm256_mul_pd(xi_144, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_147 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_104, - _mm256_set_pd( - -1.0, -1.0, -1.0, - -1.0)), - _mm256_mul_pd( - xi_99, _mm256_set_pd( - 2.0, 2.0, - 2.0, 2.0))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, - -2.0, -2.0), - _mm256_loadu_pd( - &_data_pdfs_20_33_10 - [ctr_0 + 1]))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, -2.0, - -2.0), - _mm256_loadu_pd( - &_data_pdfs_20_34_10[ctr_0 - - 1]))), - xi_103), - xi_107), - xi_143), - xi_9), - xi_96), - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0])), - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0])), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, omega_shear)); - const __m256d xi_149 = _mm256_add_pd( - _mm256_mul_pd(xi_147, _mm256_set_pd(-0.0416666666666667, - -0.0416666666666667, - -0.0416666666666667, - -0.0416666666666667)), - _mm256_mul_pd( - xi_148, _mm256_set_pd(-0.166666666666667, -0.166666666666667, - -0.166666666666667, -0.166666666666667))); - const __m256d xi_150 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_89, _mm256_set_pd(-0.1, -0.1, -0.1, -0.1)), - _mm256_mul_pd(xi_95, - _mm256_set_pd(-0.05, -0.05, -0.05, -0.05))), - xi_149); - const __m256d xi_151 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_92, - _mm256_set_pd(0.0285714285714286, - 0.0285714285714286, - 0.0285714285714286, - 0.0285714285714286)), - _mm256_mul_pd(xi_98, - _mm256_set_pd(0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143))), - xi_141), - xi_145), - xi_146), - xi_150); - const __m256d xi_166 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_147, - _mm256_set_pd(0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333)), - _mm256_mul_pd( - xi_148, _mm256_set_pd( - 0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333))), - _mm256_mul_pd(xi_92, _mm256_set_pd(-0.0714285714285714, - -0.0714285714285714, - -0.0714285714285714, - -0.0714285714285714))), - _mm256_mul_pd(xi_98, _mm256_set_pd(-0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857))), - xi_146); - const __m256d xi_171 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(rho, u_2), - _mm256_mul_pd( - vel2Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_101), - xi_106), - xi_168), - xi_6), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])); - const __m256d xi_172 = _mm256_mul_pd( - xi_171, _mm256_set_pd(xi_120, xi_120, xi_120, xi_120)); - const __m256d xi_180 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_112, _mm256_set_pd(0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159)), - _mm256_mul_pd( - xi_141, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - _mm256_mul_pd( - xi_145, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - _mm256_mul_pd(xi_88, - _mm256_set_pd(0.0952380952380952, - 0.0952380952380952, - 0.0952380952380952, - 0.0952380952380952))), - _mm256_mul_pd(xi_92, _mm256_set_pd(-0.0428571428571429, - -0.0428571428571429, - -0.0428571428571429, - -0.0428571428571429))), - _mm256_mul_pd(xi_98, _mm256_set_pd(-0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714))), - xi_150); - const __m256d xi_183 = _mm256_mul_pd( - xi_144, _mm256_set_pd(0.0625, 0.0625, 0.0625, 0.0625)); - const __m256d xi_188 = _mm256_add_pd( - _mm256_mul_pd( - xi_108, _mm256_set_pd(0.0416666666666667, 0.0416666666666667, - 0.0416666666666667, 0.0416666666666667)), - _mm256_mul_pd( - xi_91, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333))); - const __m256d xi_189 = _mm256_add_pd(xi_187, xi_188); - const __m256d xi_190 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_152, xi_182), xi_183), - xi_184), - xi_185), - xi_189); - const __m256d xi_192 = _mm256_add_pd( - _mm256_mul_pd( - xi_147, _mm256_set_pd(0.0208333333333333, 0.0208333333333333, - 0.0208333333333333, 0.0208333333333333)), - _mm256_mul_pd( - xi_148, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333))); - const __m256d xi_193 = _mm256_add_pd( - _mm256_mul_pd(xi_191, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_192); - const __m256d xi_194 = _mm256_add_pd(xi_167, xi_193); - const __m256d xi_200 = _mm256_add_pd(xi_191, xi_192); - const __m256d xi_201 = _mm256_add_pd(xi_165, xi_200); - const __m256d xi_202 = _mm256_add_pd( - _mm256_mul_pd(xi_187, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_188); - const __m256d xi_203 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_137, xi_182), xi_183), - xi_184), - xi_185), - xi_202); - const __m256d xi_206 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_117), xi_113), - xi_17), - _mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0])), - _mm256_set_pd(xi_196, xi_196, xi_196, xi_196)); - const __m256d xi_210 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_149, xi_204), xi_205), - xi_206), - xi_207), - xi_208), - xi_209); - const __m256d xi_219 = _mm256_mul_pd( - xi_171, _mm256_set_pd(xi_186, xi_186, xi_186, xi_186)); - const __m256d xi_221 = _mm256_add_pd(xi_219, xi_220); - const __m256d xi_222 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_212, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_216, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_214), - xi_218), - xi_221); - const __m256d xi_227 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_223, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_225, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_189), - xi_224), - xi_226); - const __m256d xi_228 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_224, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_226, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_202), - xi_223), - xi_225); - const __m256d xi_229 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_204, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd( - xi_206, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_149), - xi_205), - xi_207), - xi_208), - xi_209); - const __m256d xi_231 = - _mm256_mul_pd(xi_183, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_234 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_181, xi_188), xi_221), - xi_230), - xi_231), - xi_232), - xi_233); - const __m256d xi_236 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_153), xi_10), - xi_156), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])), - _mm256_set_pd(xi_196, xi_196, xi_196, xi_196)); - const __m256d xi_237 = _mm256_add_pd( - _mm256_mul_pd(xi_235, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_236, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))); - const __m256d xi_242 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_238, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_240, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_193), - xi_239), - xi_241); - const __m256d xi_243 = _mm256_add_pd(xi_235, xi_236); - const __m256d xi_244 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_239, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_241, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_200), - xi_238), - xi_240); - const __m256d xi_245 = _mm256_add_pd( - _mm256_mul_pd(xi_219, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_220); - const __m256d xi_246 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_214, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_218, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_212), - xi_216), - xi_245); - const __m256d xi_247 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_179, xi_188), xi_230), - xi_231), - xi_232), - xi_233), - xi_245); - const __m256d forceTerm_0 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_32, _mm256_set_pd(-1.5, -1.5, -1.5, -1.5)), - _mm256_mul_pd( - _mm256_mul_pd(xi_35, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_38, xi_38, xi_38, xi_38))), - _mm256_mul_pd( - _mm256_mul_pd(xi_40, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_38, xi_38, xi_38, xi_38))), - _mm256_mul_pd( - _mm256_mul_pd(xi_41, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_38, xi_38, xi_38, xi_38))); - const __m256d forceTerm_1 = _mm256_add_pd(xi_42, xi_49); - const __m256d forceTerm_2 = _mm256_add_pd( - _mm256_mul_pd(xi_42, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_49); - const __m256d forceTerm_3 = _mm256_add_pd( - _mm256_mul_pd(xi_50, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_53); - const __m256d forceTerm_4 = _mm256_add_pd(xi_50, xi_53); - const __m256d forceTerm_5 = _mm256_add_pd(xi_54, xi_55); - const __m256d forceTerm_6 = _mm256_add_pd( - _mm256_mul_pd(xi_54, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_55); - const __m256d forceTerm_7 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_62), xi_69); - const __m256d forceTerm_8 = - _mm256_add_pd(_mm256_add_pd(xi_59, xi_69), xi_70); - const __m256d forceTerm_9 = - _mm256_add_pd(_mm256_add_pd(xi_59, xi_62), xi_72); - const __m256d forceTerm_10 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_70), xi_72); - const __m256d forceTerm_11 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_73), xi_77); - const __m256d forceTerm_12 = - _mm256_add_pd(_mm256_add_pd(xi_71, xi_77), xi_78); - const __m256d forceTerm_13 = - _mm256_add_pd(_mm256_add_pd(xi_62, xi_80), xi_82); - const __m256d forceTerm_14 = - _mm256_add_pd(_mm256_add_pd(xi_70, xi_79), xi_82); - const __m256d forceTerm_15 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_78), xi_84); - const __m256d forceTerm_16 = - _mm256_add_pd(_mm256_add_pd(xi_71, xi_73), xi_84); - const __m256d forceTerm_17 = - _mm256_add_pd(_mm256_add_pd(xi_62, xi_79), xi_85); - const __m256d forceTerm_18 = - _mm256_add_pd(_mm256_add_pd(xi_70, xi_80), xi_85); - _mm256_store_pd( - &_data_pdfs_tmp_20_30_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_108, - _mm256_set_pd(-0.5, -0.5, - -0.5, -0.5)), - _mm256_mul_pd( - xi_112, - _mm256_set_pd( - 0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238))), - _mm256_mul_pd( - xi_88, _mm256_set_pd( - 0.142857142857143, - 0.142857142857143, - 0.142857142857143, - 0.142857142857143))), - _mm256_mul_pd( - xi_89, - _mm256_set_pd(0.2, 0.2, 0.2, 0.2))), - _mm256_mul_pd( - xi_91, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - _mm256_mul_pd( - xi_92, _mm256_set_pd(0.0857142857142857, - 0.0857142857142857, - 0.0857142857142857, - 0.0857142857142857))), - _mm256_mul_pd(xi_95, - _mm256_set_pd(0.1, 0.1, 0.1, 0.1))), - _mm256_mul_pd(xi_98, - _mm256_set_pd(0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429))), - forceTerm_0), - _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_31_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_116, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd( - xi_126, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - forceTerm_1), - xi_121), - xi_137), - xi_151), - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_32_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_121, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_2), - xi_116), - xi_126), - xi_151), - xi_152), - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_33_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_155, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_3), - xi_158), - xi_160), - xi_165), - xi_166), - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_34_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_158, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd( - xi_160, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - forceTerm_4), - xi_155), - xi_166), - xi_167), - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_35_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_170, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd( - xi_174, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - forceTerm_5), - xi_172), - xi_179), - xi_180), - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_36_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_172, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_6), - xi_170), - xi_174), - xi_180), - xi_181), - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_37_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_7, xi_190), xi_194), - xi_198), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_38_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_8, xi_190), xi_199), - xi_201), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_39_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_9, xi_194), xi_199), - xi_203), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_310_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_10, xi_198), xi_201), - xi_203), - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_311_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_11, xi_210), xi_222), - xi_227), - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_312_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_12, xi_222), xi_228), - xi_229), - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_313_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_13, xi_234), xi_237), - xi_242), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_314_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_14, xi_234), xi_243), - xi_244), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_315_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_15, xi_227), xi_229), - xi_246), - _mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_316_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_16, xi_210), xi_228), - xi_246), - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_317_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_17, xi_242), xi_243), - xi_247), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_318_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_18, xi_237), xi_244), - xi_247), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1]))); +static FUNC_PREFIX void kernel_streamCollide(double * RESTRICT const _data_force, double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, double kT, double omega_bulk, double omega_even, double omega_odd, double omega_shear, uint32_t seed, uint32_t time_step) +{ + const double xi_40 = -omega_bulk; + const double xi_51 = -omega_shear; + const double xi_52 = xi_51 + 2.0; + const double xi_53 = xi_52*0.5; + const double xi_58 = xi_52*0.0833333333333333; + const double xi_63 = xi_52*0.166666666666667; + const double xi_73 = xi_52*0.25; + const double xi_78 = xi_52*0.0416666666666667; + const double xi_105 = 2.4494897427831779; + const double xi_129 = omega_odd*0.25; + const double xi_144 = omega_odd*0.0833333333333333; + const double xi_207 = omega_shear*0.25; + const double xi_222 = omega_odd*0.0416666666666667; + const double xi_224 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_133 = rr_0*0.166666666666667; + const double xi_197 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + for (int64_t ctr_0 = 1; ctr_0 < ((_size_force_0 - 2) % (4) == 0 ? _size_force_0 - 2 : ((int64_t)((_size_force_0 - 2) / (4))+1) * (4)) + 1; ctr_0 += 4) + { + + __m256d random_7_0; + __m256d random_7_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 7, seed, random_7_0, random_7_1); + + + __m256d random_6_0; + __m256d random_6_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 6, seed, random_6_0, random_6_1); + + + __m256d random_5_0; + __m256d random_5_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 5, seed, random_5_0, random_5_1); + + + __m256d random_4_0; + __m256d random_4_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 4, seed, random_4_0, random_4_1); + + + __m256d random_3_0; + __m256d random_3_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1); + + + __m256d random_2_0; + __m256d random_2_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1); + + + __m256d random_1_0; + __m256d random_1_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1); + + + __m256d random_0_0; + __m256d random_0_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1); + + const __m256d xi_0 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1]),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + const __m256d xi_1 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_2 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_1),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])); + const __m256d xi_3 = _mm256_add_pd(_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0]),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_4 = _mm256_add_pd(xi_3,_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0])); + const __m256d xi_5 = _mm256_add_pd(xi_4,_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_6 = _mm256_add_pd(_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0]),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0])); + const __m256d xi_7 = _mm256_add_pd(xi_6,_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_8 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1]),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + const __m256d xi_9 = _mm256_add_pd(_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0]),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_10 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1]),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0])); + const __m256d xi_12 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])); + const __m256d xi_13 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1])); + const __m256d xi_14 = _mm256_add_pd(xi_12,xi_13); + const __m256d xi_15 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_16 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_17 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + const __m256d xi_18 = _mm256_add_pd(xi_16,xi_17); + const __m256d xi_19 = _mm256_add_pd(xi_15,xi_18); + const __m256d xi_20 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + const __m256d xi_21 = _mm256_add_pd(xi_20,_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + const __m256d xi_22 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0])); + const __m256d xi_23 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0])); + const __m256d xi_24 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_25 = _mm256_add_pd(_mm256_add_pd(xi_22,xi_23),xi_24); + const __m256d xi_26 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + const __m256d xi_27 = _mm256_add_pd(xi_12,xi_26); + const __m256d xi_28 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0])); + const __m256d xi_29 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0])); + const __m256d xi_30 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_24,xi_28),xi_29),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_31 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1]),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_32 = _mm256_add_pd(_mm256_add_pd(xi_31,xi_8),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_33 = _mm256_add_pd(_mm256_add_pd(xi_16,xi_21),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + const __m256d xi_34 = _mm256_add_pd(xi_26,_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])); + const __m256d xi_35 = _mm256_add_pd(xi_15,_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_36 = _mm256_add_pd(xi_34,xi_35); + const __m256d xi_37 = _mm256_add_pd(_mm256_add_pd(xi_9,_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0])); + const __m256d xi_38 = _mm256_add_pd(xi_28,_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_39 = _mm256_add_pd(_mm256_add_pd(xi_22,xi_38),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_57 = _mm256_mul_pd(_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667),_mm256_load_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_65 = _mm256_mul_pd(_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667),_mm256_load_pd(& _data_force_20_30_10[ctr_0])); + const __m256d xi_69 = _mm256_mul_pd(_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667),_mm256_load_pd(& _data_force_20_32_10[ctr_0])); + const __m256d xi_72 = _mm256_mul_pd(_mm256_set_pd(0.5,0.5,0.5,0.5),_mm256_load_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_76 = _mm256_mul_pd(_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333),_mm256_load_pd(& _data_force_20_30_10[ctr_0])); + const __m256d xi_80 = _mm256_mul_pd(_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333),_mm256_load_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_90 = _mm256_mul_pd(_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333),_mm256_load_pd(& _data_force_20_32_10[ctr_0])); + const __m256d xi_108 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_20_30_10[ctr_0])); + const __m256d xi_109 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0])),_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0]))),xi_108); + const __m256d xi_110 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0])),_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0]))),xi_109),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_111 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0])),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0]))); + const __m256d xi_112 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1])),_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1]))),xi_111); + const __m256d xi_113 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])),_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]))),_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1]))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0]))),xi_109),xi_112),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_116 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_117 = _mm256_add_pd(xi_116,xi_22); + const __m256d xi_118 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + const __m256d xi_121 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_122 = _mm256_add_pd(_mm256_add_pd(xi_121,xi_19),xi_27); + const __m256d xi_124 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_125 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_126 = _mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]))); + const __m256d xi_127 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])),_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1]))),_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-4.0,-4.0,-4.0,-4.0),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-4.0,-4.0,-4.0,-4.0),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0]))),xi_108),xi_112),xi_124),xi_125),xi_126),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_128 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_116,xi_23),xi_38),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0])),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0])); + const __m256d xi_130 = _mm256_mul_pd(xi_128,_mm256_set_pd(xi_129,xi_129,xi_129,xi_129)); + const __m256d xi_135 = _mm256_add_pd(random_5_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_140 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_141 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + const __m256d xi_142 = _mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1]))); + const __m256d xi_143 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_140,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_141),xi_142),xi_25),xi_4); + const __m256d xi_145 = _mm256_mul_pd(xi_143,_mm256_set_pd(xi_144,xi_144,xi_144,xi_144)); + const __m256d xi_146 = _mm256_add_pd(random_3_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_151 = _mm256_add_pd(random_0_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_168 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_121,xi_13),xi_34),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_169 = _mm256_mul_pd(xi_168,_mm256_set_pd(xi_129,xi_129,xi_129,xi_129)); + const __m256d xi_170 = _mm256_add_pd(random_4_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_172 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_141,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_14),xi_140),xi_142),xi_35),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + const __m256d xi_173 = _mm256_mul_pd(xi_172,_mm256_set_pd(xi_144,xi_144,xi_144,xi_144)); + const __m256d xi_174 = _mm256_add_pd(random_4_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_179 = _mm256_add_pd(_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0]),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_180 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_117,xi_179),xi_29),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0])); + const __m256d xi_181 = _mm256_mul_pd(xi_180,_mm256_set_pd(xi_129,xi_129,xi_129,xi_129)); + const __m256d xi_184 = _mm256_add_pd(random_5_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_186 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_124,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_125,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_126),xi_30),xi_6); + const __m256d xi_187 = _mm256_mul_pd(xi_186,_mm256_set_pd(xi_144,xi_144,xi_144,xi_144)); + const __m256d xi_188 = _mm256_add_pd(random_3_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_195 = _mm256_mul_pd(xi_127,_mm256_set_pd(0.0138888888888889,0.0138888888888889,0.0138888888888889,0.0138888888888889)); + const __m256d xi_216 = _mm256_mul_pd(xi_113,_mm256_set_pd(-0.00714285714285714,-0.00714285714285714,-0.00714285714285714,-0.00714285714285714)); + const __m256d xi_218 = _mm256_mul_pd(xi_110,_mm256_set_pd(0.025,0.025,0.025,0.025)); + const __m256d xi_223 = _mm256_mul_pd(xi_186,_mm256_set_pd(xi_222,xi_222,xi_222,xi_222)); + const __m256d xi_225 = _mm256_mul_pd(xi_180,_mm256_set_pd(xi_224,xi_224,xi_224,xi_224)); + const __m256d xi_234 = _mm256_mul_pd(xi_143,_mm256_set_pd(xi_222,xi_222,xi_222,xi_222)); + const __m256d xi_235 = _mm256_mul_pd(xi_128,_mm256_set_pd(xi_224,xi_224,xi_224,xi_224)); + const __m256d xi_243 = _mm256_mul_pd(xi_113,_mm256_set_pd(0.0178571428571429,0.0178571428571429,0.0178571428571429,0.0178571428571429)); + const __m256d xi_249 = _mm256_mul_pd(xi_168,_mm256_set_pd(xi_224,xi_224,xi_224,xi_224)); + const __m256d xi_250 = _mm256_mul_pd(xi_172,_mm256_set_pd(xi_222,xi_222,xi_222,xi_222)); + const __m256d vel0Term = xi_2; + const __m256d vel1Term = xi_5; + const __m256d vel2Term = xi_7; + const __m256d rho = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel0Term,vel1Term),vel2Term),xi_10),xi_8),xi_9),_mm256_load_pd(& _data_pdfs_20_30_10[ctr_0])); + const __m256d xi_11 = _mm256_div_pd(_mm256_set_pd(1.0,1.0,1.0,1.0),rho); + const __m256d xi_101 = _mm256_mul_pd(rho,_mm256_set_pd(kT,kT,kT,kT)); + const __m256d xi_102 = _mm256_sqrt_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0,-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0,-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0,-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0))); + const __m256d xi_103 = _mm256_mul_pd(_mm256_mul_pd(xi_102,_mm256_add_pd(random_6_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_set_pd(3.7416573867739413,3.7416573867739413,3.7416573867739413,3.7416573867739413)); + const __m256d xi_104 = _mm256_mul_pd(_mm256_mul_pd(xi_102,_mm256_add_pd(random_7_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_set_pd(5.4772255750516612,5.4772255750516612,5.4772255750516612,5.4772255750516612)); + const __m256d xi_106 = _mm256_mul_pd(_mm256_mul_pd(_mm256_sqrt_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0,-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0,-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0,-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0))),_mm256_add_pd(random_2_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_set_pd(xi_105,xi_105,xi_105,xi_105)); + const __m256d xi_107 = _mm256_mul_pd(_mm256_mul_pd(xi_102,_mm256_add_pd(random_6_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_set_pd(8.3666002653407556,8.3666002653407556,8.3666002653407556,8.3666002653407556)); + const __m256d xi_136 = _mm256_sqrt_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0,-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0,-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0,-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0))); + const __m256d xi_137 = _mm256_mul_pd(xi_136,_mm256_set_pd(1.4142135623730951,1.4142135623730951,1.4142135623730951,1.4142135623730951)); + const __m256d xi_138 = _mm256_mul_pd(xi_137,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_139 = _mm256_mul_pd(xi_135,xi_138); + const __m256d xi_147 = _mm256_mul_pd(xi_136,_mm256_set_pd(xi_105,xi_105,xi_105,xi_105)); + const __m256d xi_148 = _mm256_mul_pd(xi_147,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_149 = _mm256_mul_pd(xi_146,xi_148); + const __m256d xi_150 = _mm256_add_pd(_mm256_mul_pd(xi_145,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_149,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))); + const __m256d xi_152 = _mm256_sqrt_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0,-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0,-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0,-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0))); + const __m256d xi_153 = _mm256_mul_pd(xi_152,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_154 = _mm256_mul_pd(xi_151,xi_153); + const __m256d xi_158 = _mm256_add_pd(_mm256_mul_pd(xi_103,_mm256_set_pd(-0.119047619047619,-0.119047619047619,-0.119047619047619,-0.119047619047619)),_mm256_mul_pd(xi_127,_mm256_set_pd(-0.0198412698412698,-0.0198412698412698,-0.0198412698412698,-0.0198412698412698))); + const __m256d xi_160 = _mm256_mul_pd(_mm256_mul_pd(xi_152,_mm256_add_pd(random_0_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_set_pd(1.7320508075688772,1.7320508075688772,1.7320508075688772,1.7320508075688772)); + const __m256d xi_164 = _mm256_add_pd(xi_145,xi_149); + const __m256d xi_171 = _mm256_mul_pd(xi_138,xi_170); + const __m256d xi_175 = _mm256_mul_pd(xi_148,xi_174); + const __m256d xi_176 = _mm256_add_pd(xi_173,xi_175); + const __m256d xi_178 = _mm256_add_pd(_mm256_mul_pd(xi_173,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_175,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))); + const __m256d xi_185 = _mm256_mul_pd(xi_138,xi_184); + const __m256d xi_189 = _mm256_mul_pd(xi_148,xi_188); + const __m256d xi_190 = _mm256_add_pd(_mm256_mul_pd(xi_187,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_189,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))); + const __m256d xi_192 = _mm256_add_pd(xi_187,xi_189); + const __m256d xi_193 = _mm256_mul_pd(_mm256_mul_pd(xi_151,xi_152),_mm256_set_pd(0.25,0.25,0.25,0.25)); + const __m256d xi_196 = _mm256_mul_pd(xi_103,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_206 = _mm256_mul_pd(xi_153,_mm256_add_pd(random_1_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))); + const __m256d xi_215 = _mm256_mul_pd(xi_153,_mm256_add_pd(random_2_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))); + const __m256d xi_219 = _mm256_mul_pd(xi_107,_mm256_set_pd(-0.0142857142857143,-0.0142857142857143,-0.0142857142857143,-0.0142857142857143)); + const __m256d xi_220 = _mm256_mul_pd(xi_104,_mm256_set_pd(0.05,0.05,0.05,0.05)); + const __m256d xi_226 = _mm256_mul_pd(xi_147,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_227 = _mm256_mul_pd(xi_188,xi_226); + const __m256d xi_228 = _mm256_mul_pd(xi_137,_mm256_set_pd(0.25,0.25,0.25,0.25)); + const __m256d xi_229 = _mm256_mul_pd(xi_184,xi_228); + const __m256d xi_231 = _mm256_add_pd(_mm256_mul_pd(xi_103,_mm256_set_pd(-0.0238095238095238,-0.0238095238095238,-0.0238095238095238,-0.0238095238095238)),_mm256_mul_pd(xi_127,_mm256_set_pd(-0.00396825396825397,-0.00396825396825397,-0.00396825396825397,-0.00396825396825397))); + const __m256d xi_236 = _mm256_mul_pd(xi_146,xi_226); + const __m256d xi_237 = _mm256_mul_pd(xi_135,xi_228); + const __m256d xi_241 = _mm256_mul_pd(xi_193,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_244 = _mm256_mul_pd(xi_107,_mm256_set_pd(0.0357142857142857,0.0357142857142857,0.0357142857142857,0.0357142857142857)); + const __m256d xi_246 = _mm256_mul_pd(xi_153,_mm256_add_pd(random_1_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))); + const __m256d xi_251 = _mm256_mul_pd(xi_170,xi_228); + const __m256d xi_252 = _mm256_mul_pd(xi_174,xi_226); + const __m256d u_0 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(vel0Term,xi_14),xi_19)); + const __m256d xi_41 = _mm256_mul_pd(u_0,_mm256_load_pd(& _data_force_20_30_10[ctr_0])); + const __m256d xi_42 = _mm256_mul_pd(xi_41,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_48 = _mm256_mul_pd(xi_42,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_114 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_0,u_0))); + const __m256d xi_165 = _mm256_mul_pd(rho,u_0); + const __m256d xi_166 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel0Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_165),xi_32); + const __m256d xi_167 = _mm256_mul_pd(xi_166,_mm256_set_pd(xi_133,xi_133,xi_133,xi_133)); + const __m256d xi_202 = _mm256_mul_pd(xi_166,_mm256_set_pd(xi_197,xi_197,xi_197,xi_197)); + const __m256d u_1 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel1Term,xi_17),xi_21),xi_25)); + const __m256d xi_43 = _mm256_mul_pd(u_1,_mm256_load_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_44 = _mm256_mul_pd(xi_43,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_49 = _mm256_mul_pd(xi_44,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_71 = _mm256_mul_pd(u_1,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_74 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_0,xi_72),_mm256_mul_pd(xi_71,_mm256_load_pd(& _data_force_20_30_10[ctr_0]))),_mm256_set_pd(xi_73,xi_73,xi_73,xi_73)); + const __m256d xi_75 = _mm256_mul_pd(xi_74,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_119 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_1,u_1))); + const __m256d xi_120 = _mm256_add_pd(_mm256_add_pd(xi_118,xi_119),xi_20); + const __m256d xi_131 = _mm256_mul_pd(rho,u_1); + const __m256d xi_132 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel1Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_118),xi_131),xi_37),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + const __m256d xi_134 = _mm256_mul_pd(xi_132,_mm256_set_pd(xi_133,xi_133,xi_133,xi_133)); + const __m256d xi_198 = _mm256_mul_pd(xi_132,_mm256_set_pd(xi_197,xi_197,xi_197,xi_197)); + const __m256d u_2 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel2Term,xi_27),xi_30),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1]))); + const __m256d xi_45 = _mm256_mul_pd(u_2,_mm256_load_pd(& _data_force_20_32_10[ctr_0])); + const __m256d xi_46 = _mm256_mul_pd(xi_45,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_47 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(xi_42,xi_44),xi_46),_mm256_set_pd(xi_40 + 2.0,xi_40 + 2.0,xi_40 + 2.0,xi_40 + 2.0)); + const __m256d xi_50 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_45,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_48),xi_49); + const __m256d xi_54 = _mm256_mul_pd(xi_46,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_55 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_43,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_48),xi_54); + const __m256d xi_56 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_41,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_49),xi_54); + const __m256d xi_59 = _mm256_mul_pd(xi_50,_mm256_set_pd(xi_58,xi_58,xi_58,xi_58)); + const __m256d xi_60 = _mm256_mul_pd(xi_59,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_61 = _mm256_mul_pd(xi_56,_mm256_set_pd(xi_58,xi_58,xi_58,xi_58)); + const __m256d xi_62 = _mm256_mul_pd(xi_61,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_64 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_55,_mm256_set_pd(xi_63,xi_63,xi_63,xi_63)),xi_60),xi_62); + const __m256d xi_66 = _mm256_mul_pd(xi_55,_mm256_set_pd(xi_58,xi_58,xi_58,xi_58)); + const __m256d xi_67 = _mm256_mul_pd(xi_66,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_68 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_56,_mm256_set_pd(xi_63,xi_63,xi_63,xi_63)),xi_60),xi_67); + const __m256d xi_70 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_50,_mm256_set_pd(xi_63,xi_63,xi_63,xi_63)),xi_62),xi_67); + const __m256d xi_77 = _mm256_add_pd(_mm256_mul_pd(xi_76,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_61); + const __m256d xi_79 = _mm256_mul_pd(_mm256_mul_pd(xi_50,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_78,xi_78,xi_78,xi_78)); + const __m256d xi_81 = _mm256_mul_pd(xi_47,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_82 = _mm256_add_pd(xi_66,xi_81); + const __m256d xi_83 = _mm256_add_pd(xi_80,xi_82); + const __m256d xi_84 = _mm256_add_pd(xi_79,xi_83); + const __m256d xi_85 = _mm256_add_pd(xi_61,xi_76); + const __m256d xi_86 = _mm256_add_pd(_mm256_mul_pd(xi_80,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_82); + const __m256d xi_87 = _mm256_add_pd(xi_79,xi_86); + const __m256d xi_88 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_2,xi_72),_mm256_mul_pd(xi_71,_mm256_load_pd(& _data_force_20_32_10[ctr_0]))),_mm256_set_pd(xi_73,xi_73,xi_73,xi_73)); + const __m256d xi_89 = _mm256_mul_pd(_mm256_mul_pd(xi_56,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_78,xi_78,xi_78,xi_78)); + const __m256d xi_91 = _mm256_add_pd(xi_59,xi_90); + const __m256d xi_92 = _mm256_add_pd(xi_89,xi_91); + const __m256d xi_93 = _mm256_mul_pd(xi_88,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_94 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_mul_pd(u_0,_mm256_set_pd(0.5,0.5,0.5,0.5)),_mm256_load_pd(& _data_force_20_32_10[ctr_0])),_mm256_mul_pd(_mm256_mul_pd(u_2,_mm256_set_pd(0.5,0.5,0.5,0.5)),_mm256_load_pd(& _data_force_20_30_10[ctr_0]))),_mm256_set_pd(xi_73,xi_73,xi_73,xi_73)); + const __m256d xi_95 = _mm256_mul_pd(xi_94,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_96 = _mm256_mul_pd(_mm256_mul_pd(xi_55,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_78,xi_78,xi_78,xi_78)); + const __m256d xi_97 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_91),xi_96); + const __m256d xi_98 = _mm256_add_pd(_mm256_mul_pd(xi_90,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_59); + const __m256d xi_99 = _mm256_add_pd(xi_89,xi_98); + const __m256d xi_100 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_96),xi_98); + const __m256d xi_115 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_2,u_2))); + const __m256d xi_123 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_114,xi_115),xi_117),xi_120),xi_122),xi_24),xi_28),_mm256_load_pd(& _data_pdfs_20_30_10[ctr_0])),_mm256_set_pd(omega_bulk,omega_bulk,omega_bulk,omega_bulk)); + const __m256d xi_155 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_115,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0])),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0])); + const __m256d xi_156 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0])),xi_1),xi_120),xi_155),xi_18),xi_23),xi_31),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_157 = _mm256_mul_pd(xi_156,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_159 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_114,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_119,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1]))),xi_111),xi_118),xi_122),xi_155),xi_20),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0])),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0])),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_161 = _mm256_add_pd(_mm256_mul_pd(xi_159,_mm256_set_pd(-0.0416666666666667,-0.0416666666666667,-0.0416666666666667,-0.0416666666666667)),_mm256_mul_pd(xi_160,_mm256_set_pd(-0.166666666666667,-0.166666666666667,-0.166666666666667,-0.166666666666667))); + const __m256d xi_162 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(-0.1,-0.1,-0.1,-0.1)),_mm256_mul_pd(xi_110,_mm256_set_pd(-0.05,-0.05,-0.05,-0.05))),xi_161); + const __m256d xi_163 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_107,_mm256_set_pd(0.0285714285714286,0.0285714285714286,0.0285714285714286,0.0285714285714286)),_mm256_mul_pd(xi_113,_mm256_set_pd(0.0142857142857143,0.0142857142857143,0.0142857142857143,0.0142857142857143))),xi_154),xi_157),xi_158),xi_162); + const __m256d xi_177 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_107,_mm256_set_pd(-0.0714285714285714,-0.0714285714285714,-0.0714285714285714,-0.0714285714285714)),_mm256_mul_pd(xi_113,_mm256_set_pd(-0.0357142857142857,-0.0357142857142857,-0.0357142857142857,-0.0357142857142857))),_mm256_mul_pd(xi_159,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333))),_mm256_mul_pd(xi_160,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333))),xi_158); + const __m256d xi_182 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(rho,u_2),_mm256_mul_pd(vel2Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_10),xi_116),xi_121),xi_179),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + const __m256d xi_183 = _mm256_mul_pd(xi_182,_mm256_set_pd(xi_133,xi_133,xi_133,xi_133)); + const __m256d xi_191 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_103,_mm256_set_pd(0.0952380952380952,0.0952380952380952,0.0952380952380952,0.0952380952380952)),_mm256_mul_pd(xi_107,_mm256_set_pd(-0.0428571428571429,-0.0428571428571429,-0.0428571428571429,-0.0428571428571429))),_mm256_mul_pd(xi_113,_mm256_set_pd(-0.0214285714285714,-0.0214285714285714,-0.0214285714285714,-0.0214285714285714))),_mm256_mul_pd(xi_127,_mm256_set_pd(0.0158730158730159,0.0158730158730159,0.0158730158730159,0.0158730158730159))),_mm256_mul_pd(xi_154,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),_mm256_mul_pd(xi_157,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_162); + const __m256d xi_194 = _mm256_mul_pd(xi_156,_mm256_set_pd(0.0625,0.0625,0.0625,0.0625)); + const __m256d xi_199 = _mm256_add_pd(_mm256_mul_pd(xi_106,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)),_mm256_mul_pd(xi_123,_mm256_set_pd(0.0416666666666667,0.0416666666666667,0.0416666666666667,0.0416666666666667))); + const __m256d xi_200 = _mm256_add_pd(xi_198,xi_199); + const __m256d xi_201 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_164,xi_193),xi_194),xi_195),xi_196),xi_200); + const __m256d xi_203 = _mm256_add_pd(_mm256_mul_pd(xi_159,_mm256_set_pd(0.0208333333333333,0.0208333333333333,0.0208333333333333,0.0208333333333333)),_mm256_mul_pd(xi_160,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333))); + const __m256d xi_204 = _mm256_add_pd(_mm256_mul_pd(xi_202,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_203); + const __m256d xi_205 = _mm256_add_pd(xi_178,xi_204); + const __m256d xi_211 = _mm256_add_pd(xi_202,xi_203); + const __m256d xi_212 = _mm256_add_pd(xi_176,xi_211); + const __m256d xi_213 = _mm256_add_pd(_mm256_mul_pd(xi_198,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_199); + const __m256d xi_214 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_150,xi_193),xi_194),xi_195),xi_196),xi_213); + const __m256d xi_230 = _mm256_mul_pd(xi_182,_mm256_set_pd(xi_197,xi_197,xi_197,xi_197)); + const __m256d xi_232 = _mm256_add_pd(xi_230,xi_231); + const __m256d xi_233 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_223,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_227,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_225),xi_229),xi_232); + const __m256d xi_238 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_234,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_236,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_200),xi_235),xi_237); + const __m256d xi_239 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_235,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_237,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_213),xi_234),xi_236); + const __m256d xi_242 = _mm256_mul_pd(xi_194,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_245 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_192,xi_199),xi_232),xi_241),xi_242),xi_243),xi_244); + const __m256d xi_253 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_249,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_251,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_204),xi_250),xi_252); + const __m256d xi_255 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_250,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_252,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_211),xi_249),xi_251); + const __m256d xi_256 = _mm256_add_pd(_mm256_mul_pd(xi_230,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_231); + const __m256d xi_257 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_225,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_229,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_223),xi_227),xi_256); + const __m256d xi_258 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_190,xi_199),xi_241),xi_242),xi_243),xi_244),xi_256); + const __m256d p_0 = _mm256_add_pd(xi_2,xi_32); + const __m256d p_1 = xi_33; + const __m256d xi_208 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_1,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_0,xi_131)),_mm256_set_pd(xi_207,xi_207,xi_207,xi_207)); + const __m256d xi_209 = _mm256_add_pd(_mm256_mul_pd(xi_206,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_208,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))); + const __m256d xi_210 = _mm256_add_pd(xi_206,xi_208); + const __m256d p_2 = xi_36; + const __m256d xi_247 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_2,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_165)),_mm256_set_pd(xi_207,xi_207,xi_207,xi_207)); + const __m256d xi_248 = _mm256_add_pd(_mm256_mul_pd(xi_246,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_247,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))); + const __m256d xi_254 = _mm256_add_pd(xi_246,xi_247); + const __m256d p_3 = xi_33; + const __m256d p_4 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_37),xi_5); + const __m256d p_5 = xi_39; + const __m256d xi_217 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_5,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_131)),_mm256_set_pd(xi_207,xi_207,xi_207,xi_207)); + const __m256d xi_221 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_161,xi_215),xi_216),xi_217),xi_218),xi_219),xi_220); + const __m256d xi_240 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_215,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_217,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_161),xi_216),xi_218),xi_219),xi_220); + const __m256d p_6 = xi_36; + const __m256d p_7 = xi_39; + const __m256d p_8 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_1,xi_10),xi_3),xi_7),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d forceTerm_0 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_47,_mm256_set_pd(-1.5,-1.5,-1.5,-1.5)),_mm256_mul_pd(_mm256_mul_pd(xi_50,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_53,xi_53,xi_53,xi_53))),_mm256_mul_pd(_mm256_mul_pd(xi_55,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_53,xi_53,xi_53,xi_53))),_mm256_mul_pd(_mm256_mul_pd(xi_56,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_53,xi_53,xi_53,xi_53))); + const __m256d forceTerm_1 = _mm256_add_pd(xi_57,xi_64); + const __m256d forceTerm_2 = _mm256_add_pd(_mm256_mul_pd(xi_57,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_64); + const __m256d forceTerm_3 = _mm256_add_pd(_mm256_mul_pd(xi_65,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_68); + const __m256d forceTerm_4 = _mm256_add_pd(xi_65,xi_68); + const __m256d forceTerm_5 = _mm256_add_pd(xi_69,xi_70); + const __m256d forceTerm_6 = _mm256_add_pd(_mm256_mul_pd(xi_69,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_70); + const __m256d forceTerm_7 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_77),xi_84); + const __m256d forceTerm_8 = _mm256_add_pd(_mm256_add_pd(xi_74,xi_84),xi_85); + const __m256d forceTerm_9 = _mm256_add_pd(_mm256_add_pd(xi_74,xi_77),xi_87); + const __m256d forceTerm_10 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_85),xi_87); + const __m256d forceTerm_11 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_88),xi_92); + const __m256d forceTerm_12 = _mm256_add_pd(_mm256_add_pd(xi_86,xi_92),xi_93); + const __m256d forceTerm_13 = _mm256_add_pd(_mm256_add_pd(xi_77,xi_95),xi_97); + const __m256d forceTerm_14 = _mm256_add_pd(_mm256_add_pd(xi_85,xi_94),xi_97); + const __m256d forceTerm_15 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_93),xi_99); + const __m256d forceTerm_16 = _mm256_add_pd(_mm256_add_pd(xi_86,xi_88),xi_99); + const __m256d forceTerm_17 = _mm256_add_pd(_mm256_add_pd(xi_100,xi_77),xi_94); + const __m256d forceTerm_18 = _mm256_add_pd(_mm256_add_pd(xi_100,xi_85),xi_95); + _mm256_store_pd(&_data_pdfs_tmp_20_30_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_103,_mm256_set_pd(0.142857142857143,0.142857142857143,0.142857142857143,0.142857142857143)),_mm256_mul_pd(xi_104,_mm256_set_pd(0.2,0.2,0.2,0.2))),_mm256_mul_pd(xi_106,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),_mm256_mul_pd(xi_107,_mm256_set_pd(0.0857142857142857,0.0857142857142857,0.0857142857142857,0.0857142857142857))),_mm256_mul_pd(xi_110,_mm256_set_pd(0.1,0.1,0.1,0.1))),_mm256_mul_pd(xi_113,_mm256_set_pd(0.0428571428571429,0.0428571428571429,0.0428571428571429,0.0428571428571429))),_mm256_mul_pd(xi_123,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_mul_pd(xi_127,_mm256_set_pd(0.0238095238095238,0.0238095238095238,0.0238095238095238,0.0238095238095238))),forceTerm_0),_mm256_load_pd(& _data_pdfs_20_30_10[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_31_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_130,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_139,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),forceTerm_1),xi_134),xi_150),xi_163),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_32_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_134,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_2),xi_130),xi_139),xi_163),xi_164),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_33_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_167,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_3),xi_169),xi_171),xi_176),xi_177),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_34_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_169,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_171,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),forceTerm_4),xi_167),xi_177),xi_178),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_35_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_181,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_185,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),forceTerm_5),xi_183),xi_190),xi_191),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_36_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_183,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_6),xi_181),xi_185),xi_191),xi_192),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_37_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_7,xi_201),xi_205),xi_209),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_38_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_8,xi_201),xi_210),xi_212),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_39_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_9,xi_205),xi_210),xi_214),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_310_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_10,xi_209),xi_212),xi_214),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_311_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_11,xi_221),xi_233),xi_238),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_312_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_12,xi_233),xi_239),xi_240),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_313_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_13,xi_245),xi_248),xi_253),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_314_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_14,xi_245),xi_254),xi_255),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_315_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_15,xi_238),xi_240),xi_257),_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_316_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_16,xi_221),xi_239),xi_257),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_317_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_17,xi_253),xi_254),xi_258),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_318_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_18,xi_248),xi_255),xi_258),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]))); + } } - } - } + } +} } -} // namespace internal_kernel_streamCollide namespace internal_kernel_collide { -static FUNC_PREFIX void -kernel_collide(double *RESTRICT const _data_force, double *RESTRICT _data_pdfs, - int64_t const _size_force_0, int64_t const _size_force_1, - int64_t const _size_force_2, int64_t const _stride_force_1, - int64_t const _stride_force_2, int64_t const _stride_force_3, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, uint32_t block_offset_0, - uint32_t block_offset_1, uint32_t block_offset_2, double kT, - double omega_bulk, double omega_even, double omega_odd, - double omega_shear, uint32_t seed, uint32_t time_step) { - const double xi_25 = -omega_bulk; - const double xi_36 = -omega_shear; - const double xi_37 = xi_36 + 2.0; - const double xi_38 = xi_37 * 0.5; - const double xi_43 = xi_37 * 0.0833333333333333; - const double xi_48 = xi_37 * 0.166666666666667; - const double xi_58 = xi_37 * 0.25; - const double xi_63 = xi_37 * 0.0416666666666667; - const double xi_90 = 2.4494897427831779; - const double xi_115 = omega_odd * 0.25; - const double xi_131 = omega_odd * 0.0833333333333333; - const double xi_196 = omega_shear * 0.25; - const double xi_211 = omega_odd * 0.0416666666666667; - const double xi_213 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_120 = rr_0 * 0.166666666666667; - const double xi_186 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) { - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_313 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_316 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_311 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_312 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_314 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_315 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_35 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_36 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_317 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_318 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) { - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_313; - double *RESTRICT _data_pdfs_20_316_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_316; - double *RESTRICT _data_pdfs_20_311_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_311; - double *RESTRICT _data_pdfs_20_310_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_20_38_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_20_312_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_312; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_20_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_314; - double *RESTRICT _data_pdfs_20_315_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_315; - double *RESTRICT _data_pdfs_20_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_35; - double *RESTRICT _data_pdfs_20_37_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_32_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_32; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_31_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_20_39_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_36; - double *RESTRICT _data_pdfs_20_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_317; - double *RESTRICT _data_pdfs_20_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_318; - for (int64_t ctr_0 = 0; - ctr_0 < ((_size_force_0) % (4) == 0 - ? _size_force_0 - : ((int64_t)((_size_force_0) / (4)) + 1) * (4)); - ctr_0 += 4) { - const __m256d xi_248 = _mm256_load_pd(&_data_force_20_31_10[ctr_0]); - const __m256d xi_249 = _mm256_load_pd(&_data_force_20_30_10[ctr_0]); - const __m256d xi_250 = _mm256_load_pd(&_data_pdfs_20_33_10[ctr_0]); - const __m256d xi_251 = _mm256_load_pd(&_data_pdfs_20_313_10[ctr_0]); - const __m256d xi_252 = _mm256_load_pd(&_data_pdfs_20_316_10[ctr_0]); - const __m256d xi_253 = _mm256_load_pd(&_data_pdfs_20_311_10[ctr_0]); - const __m256d xi_254 = _mm256_load_pd(&_data_pdfs_20_310_10[ctr_0]); - const __m256d xi_255 = _mm256_load_pd(&_data_pdfs_20_38_10[ctr_0]); - const __m256d xi_256 = _mm256_load_pd(&_data_pdfs_20_312_10[ctr_0]); - const __m256d xi_257 = _mm256_load_pd(&_data_pdfs_20_34_10[ctr_0]); - const __m256d xi_258 = _mm256_load_pd(&_data_pdfs_20_314_10[ctr_0]); - const __m256d xi_259 = _mm256_load_pd(&_data_pdfs_20_315_10[ctr_0]); - const __m256d xi_260 = _mm256_load_pd(&_data_pdfs_20_35_10[ctr_0]); - const __m256d xi_261 = _mm256_load_pd(&_data_pdfs_20_37_10[ctr_0]); - const __m256d xi_262 = _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0]); - const __m256d xi_263 = _mm256_load_pd(&_data_pdfs_20_32_10[ctr_0]); - const __m256d xi_264 = _mm256_load_pd(&_data_force_20_32_10[ctr_0]); - const __m256d xi_265 = _mm256_load_pd(&_data_pdfs_20_31_10[ctr_0]); - const __m256d xi_266 = _mm256_load_pd(&_data_pdfs_20_39_10[ctr_0]); - const __m256d xi_267 = _mm256_load_pd(&_data_pdfs_20_36_10[ctr_0]); - const __m256d xi_268 = _mm256_load_pd(&_data_pdfs_20_317_10[ctr_0]); - const __m256d xi_269 = _mm256_load_pd(&_data_pdfs_20_318_10[ctr_0]); - - __m256d random_7_0; - __m256d random_7_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 7, seed, - random_7_0, random_7_1); - - __m256d random_6_0; - __m256d random_6_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 6, seed, - random_6_0, random_6_1); - - __m256d random_5_0; - __m256d random_5_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 5, seed, - random_5_0, random_5_1); - - __m256d random_4_0; - __m256d random_4_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 4, seed, - random_4_0, random_4_1); - - __m256d random_3_0; - __m256d random_3_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, - random_3_0, random_3_1); - - __m256d random_2_0; - __m256d random_2_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, - random_2_0, random_2_1); - - __m256d random_1_0; - __m256d random_1_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, - random_1_0, random_1_1); - - __m256d random_0_0; - __m256d random_0_1; - philox_double2(time_step, - _mm256_add_epi32( - _mm256_add_epi32( - _mm256_set_epi32(block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0, - block_offset_0, block_offset_0), - _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)), - _mm256_set_epi32(ctr_0, ctr_0, ctr_0, ctr_0, ctr_0, - ctr_0, ctr_0, ctr_0)), - block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, - random_0_0, random_0_1); - - const __m256d xi_0 = _mm256_add_pd(xi_258, xi_269); - const __m256d xi_1 = _mm256_add_pd(xi_0, xi_257); - const __m256d xi_2 = - _mm256_add_pd(_mm256_add_pd(xi_253, xi_259), xi_265); - const __m256d xi_3 = _mm256_add_pd(xi_256, xi_260); - const __m256d xi_4 = _mm256_add_pd(xi_250, xi_266); - const __m256d xi_5 = _mm256_add_pd(xi_252, xi_263); - const __m256d xi_6 = _mm256_add_pd(xi_267, xi_268); - const __m256d xi_8 = - _mm256_mul_pd(xi_266, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_9 = _mm256_add_pd( - _mm256_mul_pd(xi_261, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_8); - const __m256d xi_10 = - _mm256_mul_pd(xi_268, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_11 = - _mm256_mul_pd(xi_251, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_12 = - _mm256_mul_pd(xi_250, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_13 = _mm256_add_pd(_mm256_add_pd(xi_10, xi_11), xi_12); - const __m256d xi_14 = - _mm256_mul_pd(xi_263, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_15 = - _mm256_mul_pd(xi_254, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_16 = _mm256_add_pd(xi_14, xi_15); - const __m256d xi_17 = - _mm256_mul_pd(xi_252, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_18 = - _mm256_mul_pd(xi_256, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_19 = _mm256_add_pd(xi_17, xi_18); - const __m256d xi_20 = - _mm256_mul_pd(xi_269, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_21 = _mm256_add_pd(xi_10, xi_20); - const __m256d xi_22 = - _mm256_mul_pd(xi_259, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_23 = - _mm256_mul_pd(xi_267, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_24 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_17, xi_22), xi_23), xi_253); - const __m256d xi_42 = _mm256_mul_pd( - xi_248, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_50 = _mm256_mul_pd( - xi_249, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_54 = _mm256_mul_pd( - xi_264, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_57 = - _mm256_mul_pd(xi_248, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_61 = _mm256_mul_pd( - xi_249, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_65 = _mm256_mul_pd( - xi_248, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_75 = _mm256_mul_pd( - xi_264, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_93 = - _mm256_mul_pd(xi_262, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_94 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_260, _mm256_set_pd(3.0, 3.0, 3.0, 3.0)), - _mm256_mul_pd(xi_267, _mm256_set_pd(3.0, 3.0, 3.0, 3.0))), - xi_93); - const __m256d xi_95 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_252, - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0)), - _mm256_mul_pd( - xi_253, - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0))), - _mm256_mul_pd( - xi_256, - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0))), - _mm256_mul_pd( - xi_259, _mm256_set_pd(-3.0, -3.0, -3.0, -3.0))), - _mm256_mul_pd(xi_263, - _mm256_set_pd(3.0, 3.0, 3.0, 3.0))), - _mm256_mul_pd(xi_265, _mm256_set_pd(3.0, 3.0, 3.0, 3.0))), - xi_94), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_96 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_252, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)), - _mm256_mul_pd(xi_253, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))), - _mm256_mul_pd(xi_256, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))), - _mm256_mul_pd(xi_259, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))); - const __m256d xi_97 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_250, _mm256_set_pd(5.0, 5.0, 5.0, 5.0)), - _mm256_mul_pd(xi_257, _mm256_set_pd(5.0, 5.0, 5.0, 5.0))), - xi_96); - const __m256d xi_98 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_251, _mm256_set_pd(-5.0, -5.0, - -5.0, -5.0)), - _mm256_mul_pd( - xi_258, _mm256_set_pd(-5.0, -5.0, - -5.0, -5.0))), - _mm256_mul_pd( - xi_263, - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0))), - _mm256_mul_pd( - xi_265, - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0))), - _mm256_mul_pd( - xi_268, _mm256_set_pd(-5.0, -5.0, -5.0, -5.0))), - _mm256_mul_pd(xi_269, - _mm256_set_pd(-5.0, -5.0, -5.0, -5.0))), - xi_94), - xi_97), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_101 = - _mm256_mul_pd(xi_253, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_102 = _mm256_add_pd(xi_101, xi_18); - const __m256d xi_103 = - _mm256_mul_pd(xi_255, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_106 = - _mm256_mul_pd(xi_258, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_107 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_106, xi_11), xi_15), xi_21); - const __m256d xi_109 = - _mm256_mul_pd(xi_251, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_110 = - _mm256_mul_pd(xi_258, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_111 = _mm256_add_pd( - _mm256_mul_pd(xi_268, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)), - _mm256_mul_pd(xi_269, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))); - const __m256d xi_112 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_254, - _mm256_set_pd( - -7.0, -7.0, - -7.0, - -7.0)), - _mm256_mul_pd( - xi_255, - _mm256_set_pd( - -7.0, -7.0, - -7.0, - -7.0))), - _mm256_mul_pd( - xi_260, - _mm256_set_pd( - -4.0, -4.0, - -4.0, -4.0))), - _mm256_mul_pd( - xi_261, - _mm256_set_pd( - -7.0, -7.0, -7.0, - -7.0))), - _mm256_mul_pd( - xi_263, - _mm256_set_pd(5.0, 5.0, 5.0, - 5.0))), - _mm256_mul_pd( - xi_265, - _mm256_set_pd(5.0, 5.0, 5.0, - 5.0))), - _mm256_mul_pd( - xi_266, _mm256_set_pd(-7.0, -7.0, - -7.0, -7.0))), - _mm256_mul_pd( - xi_267, - _mm256_set_pd(-4.0, -4.0, -4.0, -4.0))), - xi_109), - xi_110), - xi_111), - xi_93), - xi_97), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_113 = _mm256_add_pd(xi_101, xi_256); - const __m256d xi_114 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_113, xi_14), xi_22), - xi_252), - xi_265); - const __m256d xi_116 = _mm256_mul_pd( - xi_114, _mm256_set_pd(xi_115, xi_115, xi_115, xi_115)); - const __m256d xi_118 = _mm256_add_pd(xi_103, xi_254); - const __m256d xi_122 = - _mm256_add_pd(random_5_1, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_127 = - _mm256_mul_pd(xi_261, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_128 = - _mm256_mul_pd(xi_254, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_129 = _mm256_add_pd( - _mm256_mul_pd(xi_255, _mm256_set_pd(-2.0, -2.0, -2.0, -2.0)), - _mm256_mul_pd(xi_266, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))); - const __m256d xi_130 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_127, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_128), - xi_129), - xi_14), - xi_19), - xi_2); - const __m256d xi_132 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_131, xi_131, xi_131, xi_131)); - const __m256d xi_133 = - _mm256_add_pd(random_3_0, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_138 = - _mm256_add_pd(random_0_1, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_142 = _mm256_add_pd(xi_251, xi_268); - const __m256d xi_156 = _mm256_add_pd(xi_106, xi_251); - const __m256d xi_157 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_12, xi_156), xi_20), - xi_257), - xi_268); - const __m256d xi_158 = _mm256_mul_pd( - xi_157, _mm256_set_pd(xi_115, xi_115, xi_115, xi_115)); - const __m256d xi_159 = - _mm256_add_pd(random_4_1, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_161 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_128, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_1), - xi_127), - xi_129), - xi_13); - const __m256d xi_162 = _mm256_mul_pd( - xi_161, _mm256_set_pd(xi_131, xi_131, xi_131, xi_131)); - const __m256d xi_163 = - _mm256_add_pd(random_4_0, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_168 = _mm256_add_pd(xi_252, xi_259); - const __m256d xi_169 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_102, xi_168), xi_23), xi_260); - const __m256d xi_170 = _mm256_mul_pd( - xi_169, _mm256_set_pd(xi_115, xi_115, xi_115, xi_115)); - const __m256d xi_173 = - _mm256_add_pd(random_5_0, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_175 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_109, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_110, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_111), - xi_24), - xi_3); - const __m256d xi_176 = _mm256_mul_pd( - xi_175, _mm256_set_pd(xi_131, xi_131, xi_131, xi_131)); - const __m256d xi_177 = - _mm256_add_pd(random_3_1, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5)); - const __m256d xi_184 = _mm256_mul_pd( - xi_112, _mm256_set_pd(0.0138888888888889, 0.0138888888888889, - 0.0138888888888889, 0.0138888888888889)); - const __m256d xi_205 = _mm256_mul_pd( - xi_98, _mm256_set_pd(-0.00714285714285714, -0.00714285714285714, - -0.00714285714285714, -0.00714285714285714)); - const __m256d xi_207 = - _mm256_mul_pd(xi_95, _mm256_set_pd(0.025, 0.025, 0.025, 0.025)); - const __m256d xi_212 = _mm256_mul_pd( - xi_175, _mm256_set_pd(xi_211, xi_211, xi_211, xi_211)); - const __m256d xi_214 = _mm256_mul_pd( - xi_169, _mm256_set_pd(xi_213, xi_213, xi_213, xi_213)); - const __m256d xi_223 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_211, xi_211, xi_211, xi_211)); - const __m256d xi_224 = _mm256_mul_pd( - xi_114, _mm256_set_pd(xi_213, xi_213, xi_213, xi_213)); - const __m256d xi_232 = _mm256_mul_pd( - xi_98, _mm256_set_pd(0.0178571428571429, 0.0178571428571429, - 0.0178571428571429, 0.0178571428571429)); - const __m256d xi_238 = _mm256_mul_pd( - xi_157, _mm256_set_pd(xi_213, xi_213, xi_213, xi_213)); - const __m256d xi_239 = _mm256_mul_pd( - xi_161, _mm256_set_pd(xi_211, xi_211, xi_211, xi_211)); - const __m256d vel0Term = - _mm256_add_pd(_mm256_add_pd(xi_1, xi_254), xi_255); - const __m256d vel1Term = _mm256_add_pd(xi_2, xi_261); - const __m256d vel2Term = _mm256_add_pd(xi_251, xi_3); - const __m256d rho = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel0Term, vel1Term), - vel2Term), - xi_262), - xi_4), - xi_5), - xi_6); - const __m256d xi_7 = - _mm256_div_pd(_mm256_set_pd(1.0, 1.0, 1.0, 1.0), rho); - const __m256d xi_86 = _mm256_mul_pd(rho, _mm256_set_pd(kT, kT, kT, kT)); - const __m256d xi_87 = _mm256_sqrt_pd(_mm256_mul_pd( - xi_86, - _mm256_set_pd(-((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0, - -((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0, - -((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0, - -((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0))); - const __m256d xi_88 = _mm256_mul_pd( - _mm256_mul_pd(xi_87, - _mm256_add_pd(random_6_0, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - _mm256_set_pd(3.7416573867739413, 3.7416573867739413, - 3.7416573867739413, 3.7416573867739413)); - const __m256d xi_89 = _mm256_mul_pd( - _mm256_mul_pd(xi_87, - _mm256_add_pd(random_7_0, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - _mm256_set_pd(5.4772255750516612, 5.4772255750516612, - 5.4772255750516612, 5.4772255750516612)); - const __m256d xi_91 = _mm256_mul_pd( - _mm256_mul_pd( - _mm256_sqrt_pd(_mm256_mul_pd( - xi_86, - _mm256_set_pd(-((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0, - -((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0, - -((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0, - -((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0))), - _mm256_add_pd(random_2_1, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - _mm256_set_pd(xi_90, xi_90, xi_90, xi_90)); - const __m256d xi_92 = _mm256_mul_pd( - _mm256_mul_pd(xi_87, - _mm256_add_pd(random_6_1, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - _mm256_set_pd(8.3666002653407556, 8.3666002653407556, - 8.3666002653407556, 8.3666002653407556)); - const __m256d xi_123 = _mm256_sqrt_pd(_mm256_mul_pd( - xi_86, - _mm256_set_pd(-((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0, - -((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0, - -((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0, - -((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0))); - const __m256d xi_124 = _mm256_mul_pd( - xi_123, _mm256_set_pd(1.4142135623730951, 1.4142135623730951, - 1.4142135623730951, 1.4142135623730951)); - const __m256d xi_125 = - _mm256_mul_pd(xi_124, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_126 = _mm256_mul_pd(xi_122, xi_125); - const __m256d xi_134 = - _mm256_mul_pd(xi_123, _mm256_set_pd(xi_90, xi_90, xi_90, xi_90)); - const __m256d xi_135 = _mm256_mul_pd( - xi_134, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_136 = _mm256_mul_pd(xi_133, xi_135); - const __m256d xi_137 = _mm256_add_pd( - _mm256_mul_pd(xi_132, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_136, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))); - const __m256d xi_139 = _mm256_sqrt_pd(_mm256_mul_pd( - xi_86, _mm256_set_pd(-((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0, - -((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0, - -((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0, - -((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0))); - const __m256d xi_140 = - _mm256_mul_pd(xi_139, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_141 = _mm256_mul_pd(xi_138, xi_140); - const __m256d xi_146 = _mm256_add_pd( - _mm256_mul_pd(xi_112, _mm256_set_pd(-0.0198412698412698, - -0.0198412698412698, - -0.0198412698412698, - -0.0198412698412698)), - _mm256_mul_pd( - xi_88, _mm256_set_pd(-0.119047619047619, -0.119047619047619, - -0.119047619047619, -0.119047619047619))); - const __m256d xi_148 = _mm256_mul_pd( - _mm256_mul_pd(xi_139, - _mm256_add_pd(random_0_0, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - _mm256_set_pd(1.7320508075688772, 1.7320508075688772, - 1.7320508075688772, 1.7320508075688772)); - const __m256d xi_152 = _mm256_add_pd(xi_132, xi_136); - const __m256d xi_160 = _mm256_mul_pd(xi_125, xi_159); - const __m256d xi_164 = _mm256_mul_pd(xi_135, xi_163); - const __m256d xi_165 = _mm256_add_pd(xi_162, xi_164); - const __m256d xi_167 = _mm256_add_pd( - _mm256_mul_pd(xi_162, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_164, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))); - const __m256d xi_174 = _mm256_mul_pd(xi_125, xi_173); - const __m256d xi_178 = _mm256_mul_pd(xi_135, xi_177); - const __m256d xi_179 = _mm256_add_pd( - _mm256_mul_pd(xi_176, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_178, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))); - const __m256d xi_181 = _mm256_add_pd(xi_176, xi_178); - const __m256d xi_182 = - _mm256_mul_pd(_mm256_mul_pd(xi_138, xi_139), - _mm256_set_pd(0.25, 0.25, 0.25, 0.25)); - const __m256d xi_185 = _mm256_mul_pd( - xi_88, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_195 = _mm256_mul_pd( - xi_140, - _mm256_add_pd(random_1_0, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))); - const __m256d xi_204 = _mm256_mul_pd( - xi_140, - _mm256_add_pd(random_2_0, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))); - const __m256d xi_208 = _mm256_mul_pd( - xi_92, _mm256_set_pd(-0.0142857142857143, -0.0142857142857143, - -0.0142857142857143, -0.0142857142857143)); - const __m256d xi_209 = - _mm256_mul_pd(xi_89, _mm256_set_pd(0.05, 0.05, 0.05, 0.05)); - const __m256d xi_215 = _mm256_mul_pd( - xi_134, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_216 = _mm256_mul_pd(xi_177, xi_215); - const __m256d xi_217 = - _mm256_mul_pd(xi_124, _mm256_set_pd(0.25, 0.25, 0.25, 0.25)); - const __m256d xi_218 = _mm256_mul_pd(xi_173, xi_217); - const __m256d xi_220 = _mm256_add_pd( - _mm256_mul_pd(xi_112, _mm256_set_pd(-0.00396825396825397, - -0.00396825396825397, - -0.00396825396825397, - -0.00396825396825397)), - _mm256_mul_pd(xi_88, _mm256_set_pd(-0.0238095238095238, - -0.0238095238095238, - -0.0238095238095238, - -0.0238095238095238))); - const __m256d xi_225 = _mm256_mul_pd(xi_133, xi_215); - const __m256d xi_226 = _mm256_mul_pd(xi_122, xi_217); - const __m256d xi_230 = - _mm256_mul_pd(xi_182, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_233 = _mm256_mul_pd( - xi_92, _mm256_set_pd(0.0357142857142857, 0.0357142857142857, - 0.0357142857142857, 0.0357142857142857)); - const __m256d xi_235 = _mm256_mul_pd( - xi_140, - _mm256_add_pd(random_1_1, _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))); - const __m256d xi_240 = _mm256_mul_pd(xi_159, xi_217); - const __m256d xi_241 = _mm256_mul_pd(xi_163, xi_215); - const __m256d u_0 = _mm256_mul_pd( - xi_7, _mm256_add_pd(_mm256_add_pd(vel0Term, xi_13), xi_9)); - const __m256d xi_26 = _mm256_mul_pd(u_0, xi_249); - const __m256d xi_27 = _mm256_mul_pd( - xi_26, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_33 = - _mm256_mul_pd(xi_27, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_99 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_0, u_0))); - const __m256d xi_153 = _mm256_mul_pd(rho, u_0); - const __m256d xi_154 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(vel0Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_142), - xi_153), - xi_261), - xi_4); - const __m256d xi_155 = _mm256_mul_pd( - xi_154, _mm256_set_pd(xi_120, xi_120, xi_120, xi_120)); - const __m256d xi_191 = _mm256_mul_pd( - xi_154, _mm256_set_pd(xi_186, xi_186, xi_186, xi_186)); - const __m256d u_1 = _mm256_mul_pd( - xi_7, _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel1Term, xi_16), xi_19), - xi_255), - xi_8)); - const __m256d xi_28 = _mm256_mul_pd(u_1, xi_248); - const __m256d xi_29 = _mm256_mul_pd( - xi_28, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_34 = - _mm256_mul_pd(xi_29, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_56 = - _mm256_mul_pd(u_1, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_59 = - _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_0, xi_57), - _mm256_mul_pd(xi_249, xi_56)), - _mm256_set_pd(xi_58, xi_58, xi_58, xi_58)); - const __m256d xi_60 = - _mm256_mul_pd(xi_59, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_104 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_1, u_1))); - const __m256d xi_105 = - _mm256_add_pd(_mm256_add_pd(xi_103, xi_104), xi_9); - const __m256d xi_117 = _mm256_mul_pd(rho, u_1); - const __m256d xi_119 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(vel1Term, _mm256_set_pd(-1.0, -1.0, - -1.0, -1.0)), - xi_117), - xi_118), - xi_256), - xi_266), - xi_5); - const __m256d xi_121 = _mm256_mul_pd( - xi_119, _mm256_set_pd(xi_120, xi_120, xi_120, xi_120)); - const __m256d xi_187 = _mm256_mul_pd( - xi_119, _mm256_set_pd(xi_186, xi_186, xi_186, xi_186)); - const __m256d xi_197 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_0, xi_117), xi_118), - xi_261), - xi_8), - _mm256_set_pd(xi_196, xi_196, xi_196, xi_196)); - const __m256d xi_198 = _mm256_add_pd( - _mm256_mul_pd(xi_195, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_197, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))); - const __m256d xi_199 = _mm256_add_pd(xi_195, xi_197); - const __m256d u_2 = _mm256_mul_pd( - xi_7, - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel2Term, xi_21), xi_24), - xi_258)); - const __m256d xi_30 = _mm256_mul_pd(u_2, xi_264); - const __m256d xi_31 = _mm256_mul_pd( - xi_30, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_32 = _mm256_mul_pd( - _mm256_add_pd(_mm256_add_pd(xi_27, xi_29), xi_31), - _mm256_set_pd(xi_25 + 2.0, xi_25 + 2.0, xi_25 + 2.0, xi_25 + 2.0)); - const __m256d xi_35 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_30, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_33), - xi_34); - const __m256d xi_39 = - _mm256_mul_pd(xi_31, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_40 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_28, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_33), - xi_39); - const __m256d xi_41 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_26, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_34), - xi_39); - const __m256d xi_44 = - _mm256_mul_pd(xi_35, _mm256_set_pd(xi_43, xi_43, xi_43, xi_43)); - const __m256d xi_45 = - _mm256_mul_pd(xi_44, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_46 = - _mm256_mul_pd(xi_41, _mm256_set_pd(xi_43, xi_43, xi_43, xi_43)); - const __m256d xi_47 = - _mm256_mul_pd(xi_46, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_49 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_40, _mm256_set_pd(xi_48, xi_48, xi_48, xi_48)), - xi_45), - xi_47); - const __m256d xi_51 = - _mm256_mul_pd(xi_40, _mm256_set_pd(xi_43, xi_43, xi_43, xi_43)); - const __m256d xi_52 = - _mm256_mul_pd(xi_51, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_53 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_41, _mm256_set_pd(xi_48, xi_48, xi_48, xi_48)), - xi_45), - xi_52); - const __m256d xi_55 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_35, _mm256_set_pd(xi_48, xi_48, xi_48, xi_48)), - xi_47), - xi_52); - const __m256d xi_62 = _mm256_add_pd( - _mm256_mul_pd(xi_61, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_46); - const __m256d xi_64 = _mm256_mul_pd( - _mm256_mul_pd(xi_35, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_63, xi_63, xi_63, xi_63)); - const __m256d xi_66 = - _mm256_mul_pd(xi_32, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_67 = _mm256_add_pd(xi_51, xi_66); - const __m256d xi_68 = _mm256_add_pd(xi_65, xi_67); - const __m256d xi_69 = _mm256_add_pd(xi_64, xi_68); - const __m256d xi_70 = _mm256_add_pd(xi_46, xi_61); - const __m256d xi_71 = _mm256_add_pd( - _mm256_mul_pd(xi_65, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_67); - const __m256d xi_72 = _mm256_add_pd(xi_64, xi_71); - const __m256d xi_73 = - _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_57), - _mm256_mul_pd(xi_264, xi_56)), - _mm256_set_pd(xi_58, xi_58, xi_58, xi_58)); - const __m256d xi_74 = _mm256_mul_pd( - _mm256_mul_pd(xi_41, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_63, xi_63, xi_63, xi_63)); - const __m256d xi_76 = _mm256_add_pd(xi_44, xi_75); - const __m256d xi_77 = _mm256_add_pd(xi_74, xi_76); - const __m256d xi_78 = - _mm256_mul_pd(xi_73, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_79 = _mm256_mul_pd( - _mm256_add_pd(_mm256_mul_pd(_mm256_mul_pd(u_0, xi_264), - _mm256_set_pd(0.5, 0.5, 0.5, 0.5)), - _mm256_mul_pd(_mm256_mul_pd(u_2, xi_249), - _mm256_set_pd(0.5, 0.5, 0.5, 0.5))), - _mm256_set_pd(xi_58, xi_58, xi_58, xi_58)); - const __m256d xi_80 = - _mm256_mul_pd(xi_79, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_81 = _mm256_mul_pd( - _mm256_mul_pd(xi_40, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_63, xi_63, xi_63, xi_63)); - const __m256d xi_82 = _mm256_add_pd(_mm256_add_pd(xi_66, xi_76), xi_81); - const __m256d xi_83 = _mm256_add_pd( - _mm256_mul_pd(xi_75, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_44); - const __m256d xi_84 = _mm256_add_pd(xi_74, xi_83); - const __m256d xi_85 = _mm256_add_pd(_mm256_add_pd(xi_66, xi_81), xi_83); - const __m256d xi_100 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_2, u_2))); - const __m256d xi_108 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_100, xi_102), - xi_105), - xi_107), - xi_17), - xi_22), - xi_262), - xi_99), - _mm256_set_pd(omega_bulk, omega_bulk, omega_bulk, omega_bulk)); - const __m256d xi_143 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_100, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_260), - xi_267); - const __m256d xi_144 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_265, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_0), - xi_105), - xi_142), - xi_143), - xi_16), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, omega_shear)); - const __m256d xi_145 = - _mm256_mul_pd(xi_144, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_147 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_104, - _mm256_set_pd( - -1.0, -1.0, -1.0, - -1.0)), - _mm256_mul_pd( - xi_250, - _mm256_set_pd( - -2.0, -2.0, -2.0, - -2.0))), - _mm256_mul_pd( - xi_257, - _mm256_set_pd(-2.0, -2.0, - -2.0, -2.0))), - _mm256_mul_pd( - xi_99, - _mm256_set_pd(2.0, 2.0, 2.0, - 2.0))), - xi_103), - xi_107), - xi_143), - xi_263), - xi_265), - xi_9), - xi_96), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, omega_shear)); - const __m256d xi_149 = _mm256_add_pd( - _mm256_mul_pd(xi_147, _mm256_set_pd(-0.0416666666666667, - -0.0416666666666667, - -0.0416666666666667, - -0.0416666666666667)), - _mm256_mul_pd( - xi_148, _mm256_set_pd(-0.166666666666667, -0.166666666666667, - -0.166666666666667, -0.166666666666667))); - const __m256d xi_150 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_89, _mm256_set_pd(-0.1, -0.1, -0.1, -0.1)), - _mm256_mul_pd(xi_95, - _mm256_set_pd(-0.05, -0.05, -0.05, -0.05))), - xi_149); - const __m256d xi_151 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_92, - _mm256_set_pd(0.0285714285714286, - 0.0285714285714286, - 0.0285714285714286, - 0.0285714285714286)), - _mm256_mul_pd(xi_98, - _mm256_set_pd(0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143))), - xi_141), - xi_145), - xi_146), - xi_150); - const __m256d xi_166 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_147, - _mm256_set_pd(0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333)), - _mm256_mul_pd( - xi_148, _mm256_set_pd( - 0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333))), - _mm256_mul_pd(xi_92, _mm256_set_pd(-0.0714285714285714, - -0.0714285714285714, - -0.0714285714285714, - -0.0714285714285714))), - _mm256_mul_pd(xi_98, _mm256_set_pd(-0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857))), - xi_146); - const __m256d xi_171 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(rho, u_2), - _mm256_mul_pd( - vel2Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_101), - xi_106), - xi_168), - xi_269), - xi_6); - const __m256d xi_172 = _mm256_mul_pd( - xi_171, _mm256_set_pd(xi_120, xi_120, xi_120, xi_120)); - const __m256d xi_180 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_112, _mm256_set_pd(0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159)), - _mm256_mul_pd( - xi_141, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - _mm256_mul_pd( - xi_145, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - _mm256_mul_pd(xi_88, - _mm256_set_pd(0.0952380952380952, - 0.0952380952380952, - 0.0952380952380952, - 0.0952380952380952))), - _mm256_mul_pd(xi_92, _mm256_set_pd(-0.0428571428571429, - -0.0428571428571429, - -0.0428571428571429, - -0.0428571428571429))), - _mm256_mul_pd(xi_98, _mm256_set_pd(-0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714))), - xi_150); - const __m256d xi_183 = _mm256_mul_pd( - xi_144, _mm256_set_pd(0.0625, 0.0625, 0.0625, 0.0625)); - const __m256d xi_188 = _mm256_add_pd( - _mm256_mul_pd( - xi_108, _mm256_set_pd(0.0416666666666667, 0.0416666666666667, - 0.0416666666666667, 0.0416666666666667)), - _mm256_mul_pd( - xi_91, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333))); - const __m256d xi_189 = _mm256_add_pd(xi_187, xi_188); - const __m256d xi_190 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_152, xi_182), xi_183), - xi_184), - xi_185), - xi_189); - const __m256d xi_192 = _mm256_add_pd( - _mm256_mul_pd( - xi_147, _mm256_set_pd(0.0208333333333333, 0.0208333333333333, - 0.0208333333333333, 0.0208333333333333)), - _mm256_mul_pd( - xi_148, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333))); - const __m256d xi_193 = _mm256_add_pd( - _mm256_mul_pd(xi_191, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_192); - const __m256d xi_194 = _mm256_add_pd(xi_167, xi_193); - const __m256d xi_200 = _mm256_add_pd(xi_191, xi_192); - const __m256d xi_201 = _mm256_add_pd(xi_165, xi_200); - const __m256d xi_202 = _mm256_add_pd( - _mm256_mul_pd(xi_187, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_188); - const __m256d xi_203 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_137, xi_182), xi_183), - xi_184), - xi_185), - xi_202); - const __m256d xi_206 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_117), xi_113), - xi_17), - xi_259), - _mm256_set_pd(xi_196, xi_196, xi_196, xi_196)); - const __m256d xi_210 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_149, xi_204), xi_205), - xi_206), - xi_207), - xi_208), - xi_209); - const __m256d xi_219 = _mm256_mul_pd( - xi_171, _mm256_set_pd(xi_186, xi_186, xi_186, xi_186)); - const __m256d xi_221 = _mm256_add_pd(xi_219, xi_220); - const __m256d xi_222 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_212, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_216, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_214), - xi_218), - xi_221); - const __m256d xi_227 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_223, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_225, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_189), - xi_224), - xi_226); - const __m256d xi_228 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_224, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_226, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_202), - xi_223), - xi_225); - const __m256d xi_229 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_204, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd( - xi_206, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_149), - xi_205), - xi_207), - xi_208), - xi_209); - const __m256d xi_231 = - _mm256_mul_pd(xi_183, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_234 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_181, xi_188), xi_221), - xi_230), - xi_231), - xi_232), - xi_233); - const __m256d xi_236 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_153), xi_10), - xi_156), - xi_269), - _mm256_set_pd(xi_196, xi_196, xi_196, xi_196)); - const __m256d xi_237 = _mm256_add_pd( - _mm256_mul_pd(xi_235, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_236, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))); - const __m256d xi_242 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_238, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_240, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_193), - xi_239), - xi_241); - const __m256d xi_243 = _mm256_add_pd(xi_235, xi_236); - const __m256d xi_244 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_239, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_241, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_200), - xi_238), - xi_240); - const __m256d xi_245 = _mm256_add_pd( - _mm256_mul_pd(xi_219, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_220); - const __m256d xi_246 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_214, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_218, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_212), - xi_216), - xi_245); - const __m256d xi_247 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_179, xi_188), xi_230), - xi_231), - xi_232), - xi_233), - xi_245); - const __m256d forceTerm_0 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_32, _mm256_set_pd(-1.5, -1.5, -1.5, -1.5)), - _mm256_mul_pd( - _mm256_mul_pd(xi_35, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_38, xi_38, xi_38, xi_38))), - _mm256_mul_pd( - _mm256_mul_pd(xi_40, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_38, xi_38, xi_38, xi_38))), - _mm256_mul_pd( - _mm256_mul_pd(xi_41, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_38, xi_38, xi_38, xi_38))); - const __m256d forceTerm_1 = _mm256_add_pd(xi_42, xi_49); - const __m256d forceTerm_2 = _mm256_add_pd( - _mm256_mul_pd(xi_42, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_49); - const __m256d forceTerm_3 = _mm256_add_pd( - _mm256_mul_pd(xi_50, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_53); - const __m256d forceTerm_4 = _mm256_add_pd(xi_50, xi_53); - const __m256d forceTerm_5 = _mm256_add_pd(xi_54, xi_55); - const __m256d forceTerm_6 = _mm256_add_pd( - _mm256_mul_pd(xi_54, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_55); - const __m256d forceTerm_7 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_62), xi_69); - const __m256d forceTerm_8 = - _mm256_add_pd(_mm256_add_pd(xi_59, xi_69), xi_70); - const __m256d forceTerm_9 = - _mm256_add_pd(_mm256_add_pd(xi_59, xi_62), xi_72); - const __m256d forceTerm_10 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_70), xi_72); - const __m256d forceTerm_11 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_73), xi_77); - const __m256d forceTerm_12 = - _mm256_add_pd(_mm256_add_pd(xi_71, xi_77), xi_78); - const __m256d forceTerm_13 = - _mm256_add_pd(_mm256_add_pd(xi_62, xi_80), xi_82); - const __m256d forceTerm_14 = - _mm256_add_pd(_mm256_add_pd(xi_70, xi_79), xi_82); - const __m256d forceTerm_15 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_78), xi_84); - const __m256d forceTerm_16 = - _mm256_add_pd(_mm256_add_pd(xi_71, xi_73), xi_84); - const __m256d forceTerm_17 = - _mm256_add_pd(_mm256_add_pd(xi_62, xi_79), xi_85); - const __m256d forceTerm_18 = - _mm256_add_pd(_mm256_add_pd(xi_70, xi_80), xi_85); - _mm256_store_pd( - &_data_pdfs_20_30_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_108, - _mm256_set_pd(-0.5, -0.5, - -0.5, -0.5)), - _mm256_mul_pd( - xi_112, - _mm256_set_pd( - 0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238))), - _mm256_mul_pd( - xi_88, _mm256_set_pd( - 0.142857142857143, - 0.142857142857143, - 0.142857142857143, - 0.142857142857143))), - _mm256_mul_pd( - xi_89, - _mm256_set_pd(0.2, 0.2, 0.2, 0.2))), - _mm256_mul_pd( - xi_91, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - _mm256_mul_pd( - xi_92, _mm256_set_pd(0.0857142857142857, - 0.0857142857142857, - 0.0857142857142857, - 0.0857142857142857))), - _mm256_mul_pd(xi_95, - _mm256_set_pd(0.1, 0.1, 0.1, 0.1))), - _mm256_mul_pd(xi_98, - _mm256_set_pd(0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429))), - forceTerm_0), - xi_262)); - _mm256_store_pd( - &_data_pdfs_20_31_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_116, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd( - xi_126, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - forceTerm_1), - xi_121), - xi_137), - xi_151), - xi_265)); - _mm256_store_pd( - &_data_pdfs_20_32_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_121, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_2), - xi_116), - xi_126), - xi_151), - xi_152), - xi_263)); - _mm256_store_pd( - &_data_pdfs_20_33_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_155, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_3), - xi_158), - xi_160), - xi_165), - xi_166), - xi_250)); - _mm256_store_pd( - &_data_pdfs_20_34_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_158, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd( - xi_160, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - forceTerm_4), - xi_155), - xi_166), - xi_167), - xi_257)); - _mm256_store_pd( - &_data_pdfs_20_35_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_170, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd( - xi_174, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - forceTerm_5), - xi_172), - xi_179), - xi_180), - xi_260)); - _mm256_store_pd( - &_data_pdfs_20_36_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_172, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_6), - xi_170), - xi_174), - xi_180), - xi_181), - xi_267)); - _mm256_store_pd( - &_data_pdfs_20_37_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_7, xi_190), xi_194), - xi_198), - xi_261)); - _mm256_store_pd( - &_data_pdfs_20_38_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_8, xi_190), xi_199), - xi_201), - xi_255)); - _mm256_store_pd( - &_data_pdfs_20_39_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_9, xi_194), xi_199), - xi_203), - xi_266)); - _mm256_store_pd( - &_data_pdfs_20_310_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_10, xi_198), xi_201), - xi_203), - xi_254)); - _mm256_store_pd( - &_data_pdfs_20_311_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_11, xi_210), xi_222), - xi_227), - xi_253)); - _mm256_store_pd( - &_data_pdfs_20_312_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_12, xi_222), xi_228), - xi_229), - xi_256)); - _mm256_store_pd( - &_data_pdfs_20_313_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_13, xi_234), xi_237), - xi_242), - xi_251)); - _mm256_store_pd( - &_data_pdfs_20_314_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_14, xi_234), xi_243), - xi_244), - xi_258)); - _mm256_store_pd( - &_data_pdfs_20_315_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_15, xi_227), xi_229), - xi_246), - xi_259)); - _mm256_store_pd( - &_data_pdfs_20_316_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_16, xi_210), xi_228), - xi_246), - xi_252)); - _mm256_store_pd( - &_data_pdfs_20_317_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_17, xi_242), xi_243), - xi_247), - xi_268)); - _mm256_store_pd( - &_data_pdfs_20_318_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_18, xi_237), xi_244), - xi_247), - xi_269)); +static FUNC_PREFIX void kernel_collide(double * RESTRICT const _data_force, double * RESTRICT _data_pdfs, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, double kT, double omega_bulk, double omega_even, double omega_odd, double omega_shear, uint32_t seed, uint32_t time_step) +{ + const double xi_40 = -omega_bulk; + const double xi_51 = -omega_shear; + const double xi_52 = xi_51 + 2.0; + const double xi_53 = xi_52*0.5; + const double xi_58 = xi_52*0.0833333333333333; + const double xi_63 = xi_52*0.166666666666667; + const double xi_73 = xi_52*0.25; + const double xi_78 = xi_52*0.0416666666666667; + const double xi_105 = 2.4494897427831779; + const double xi_129 = omega_odd*0.25; + const double xi_144 = omega_odd*0.0833333333333333; + const double xi_207 = omega_shear*0.25; + const double xi_222 = omega_odd*0.0416666666666667; + const double xi_224 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_133 = rr_0*0.166666666666667; + const double xi_197 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_20_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_pdfs_20_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + 16*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_38_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_20_31_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_35; + double * RESTRICT _data_pdfs_20_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_318; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_20_311_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_311; + double * RESTRICT _data_pdfs_20_310_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_20_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_36; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_37_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_20_315_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_315; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_pdfs_20_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_314; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_39_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_313; + double * RESTRICT _data_pdfs_20_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_317; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_pdfs_20_32_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_20_312_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_312; + double * RESTRICT _data_pdfs_20_316_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_316; + for (int64_t ctr_0 = 0; ctr_0 < ((_size_force_0) % (4) == 0 ? _size_force_0 : ((int64_t)((_size_force_0) / (4))+1) * (4)); ctr_0 += 4) + { + const __m256d xi_259 = _mm256_load_pd(& _data_pdfs_20_38_10[ctr_0]); + const __m256d xi_260 = _mm256_load_pd(& _data_pdfs_20_31_10[ctr_0]); + const __m256d xi_261 = _mm256_load_pd(& _data_pdfs_20_35_10[ctr_0]); + const __m256d xi_262 = _mm256_load_pd(& _data_pdfs_20_318_10[ctr_0]); + const __m256d xi_263 = _mm256_load_pd(& _data_pdfs_20_30_10[ctr_0]); + const __m256d xi_264 = _mm256_load_pd(& _data_pdfs_20_311_10[ctr_0]); + const __m256d xi_265 = _mm256_load_pd(& _data_pdfs_20_310_10[ctr_0]); + const __m256d xi_266 = _mm256_load_pd(& _data_pdfs_20_34_10[ctr_0]); + const __m256d xi_267 = _mm256_load_pd(& _data_pdfs_20_36_10[ctr_0]); + const __m256d xi_268 = _mm256_load_pd(& _data_pdfs_20_33_10[ctr_0]); + const __m256d xi_269 = _mm256_load_pd(& _data_pdfs_20_37_10[ctr_0]); + const __m256d xi_270 = _mm256_load_pd(& _data_pdfs_20_315_10[ctr_0]); + const __m256d xi_271 = _mm256_load_pd(& _data_force_20_31_10[ctr_0]); + const __m256d xi_272 = _mm256_load_pd(& _data_pdfs_20_314_10[ctr_0]); + const __m256d xi_273 = _mm256_load_pd(& _data_force_20_32_10[ctr_0]); + const __m256d xi_274 = _mm256_load_pd(& _data_pdfs_20_39_10[ctr_0]); + const __m256d xi_275 = _mm256_load_pd(& _data_pdfs_20_313_10[ctr_0]); + const __m256d xi_276 = _mm256_load_pd(& _data_pdfs_20_317_10[ctr_0]); + const __m256d xi_277 = _mm256_load_pd(& _data_force_20_30_10[ctr_0]); + const __m256d xi_278 = _mm256_load_pd(& _data_pdfs_20_32_10[ctr_0]); + const __m256d xi_279 = _mm256_load_pd(& _data_pdfs_20_312_10[ctr_0]); + const __m256d xi_280 = _mm256_load_pd(& _data_pdfs_20_316_10[ctr_0]); + + __m256d random_7_0; + __m256d random_7_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 7, seed, random_7_0, random_7_1); + + + __m256d random_6_0; + __m256d random_6_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 6, seed, random_6_0, random_6_1); + + + __m256d random_5_0; + __m256d random_5_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 5, seed, random_5_0, random_5_1); + + + __m256d random_4_0; + __m256d random_4_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 4, seed, random_4_0, random_4_1); + + + __m256d random_3_0; + __m256d random_3_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1); + + + __m256d random_2_0; + __m256d random_2_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1); + + + __m256d random_1_0; + __m256d random_1_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1); + + + __m256d random_0_0; + __m256d random_0_1; + philox_double2(time_step, _mm256_add_epi32(_mm256_add_epi32(_mm256_set_epi32(block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0,block_offset_0), _mm256_set_epi32(7,6,5,4,3,2,1,0)), _mm256_set_epi32(ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0,ctr_0)), block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1); + + const __m256d xi_0 = _mm256_add_pd(xi_259,xi_265); + const __m256d xi_1 = _mm256_add_pd(xi_262,xi_272); + const __m256d xi_2 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_1),xi_266); + const __m256d xi_3 = _mm256_add_pd(xi_264,xi_270); + const __m256d xi_4 = _mm256_add_pd(xi_260,xi_3); + const __m256d xi_5 = _mm256_add_pd(xi_269,xi_4); + const __m256d xi_6 = _mm256_add_pd(xi_261,xi_279); + const __m256d xi_7 = _mm256_add_pd(xi_275,xi_6); + const __m256d xi_8 = _mm256_add_pd(xi_268,xi_274); + const __m256d xi_9 = _mm256_add_pd(xi_278,xi_280); + const __m256d xi_10 = _mm256_add_pd(xi_267,xi_276); + const __m256d xi_12 = _mm256_mul_pd(xi_276,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_13 = _mm256_mul_pd(xi_268,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_14 = _mm256_add_pd(xi_12,xi_13); + const __m256d xi_15 = _mm256_mul_pd(xi_275,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_16 = _mm256_mul_pd(xi_269,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_17 = _mm256_mul_pd(xi_274,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_18 = _mm256_add_pd(xi_16,xi_17); + const __m256d xi_19 = _mm256_add_pd(xi_15,xi_18); + const __m256d xi_20 = _mm256_mul_pd(xi_265,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_21 = _mm256_add_pd(xi_20,xi_259); + const __m256d xi_22 = _mm256_mul_pd(xi_279,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_23 = _mm256_mul_pd(xi_278,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_24 = _mm256_mul_pd(xi_280,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_25 = _mm256_add_pd(_mm256_add_pd(xi_22,xi_23),xi_24); + const __m256d xi_26 = _mm256_mul_pd(xi_262,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_27 = _mm256_add_pd(xi_12,xi_26); + const __m256d xi_28 = _mm256_mul_pd(xi_270,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_29 = _mm256_mul_pd(xi_267,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_30 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_24,xi_264),xi_28),xi_29); + const __m256d xi_31 = _mm256_add_pd(xi_275,xi_276); + const __m256d xi_32 = _mm256_add_pd(_mm256_add_pd(xi_269,xi_31),xi_8); + const __m256d xi_33 = _mm256_add_pd(_mm256_add_pd(xi_16,xi_21),xi_274); + const __m256d xi_34 = _mm256_add_pd(xi_26,xi_276); + const __m256d xi_35 = _mm256_add_pd(xi_15,xi_272); + const __m256d xi_36 = _mm256_add_pd(xi_34,xi_35); + const __m256d xi_37 = _mm256_add_pd(_mm256_add_pd(xi_274,xi_279),xi_9); + const __m256d xi_38 = _mm256_add_pd(xi_28,xi_280); + const __m256d xi_39 = _mm256_add_pd(_mm256_add_pd(xi_22,xi_264),xi_38); + const __m256d xi_57 = _mm256_mul_pd(xi_271,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_65 = _mm256_mul_pd(xi_277,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_69 = _mm256_mul_pd(xi_273,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_72 = _mm256_mul_pd(xi_271,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_76 = _mm256_mul_pd(xi_277,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_80 = _mm256_mul_pd(xi_271,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_90 = _mm256_mul_pd(xi_273,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_108 = _mm256_mul_pd(xi_263,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_109 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_261,_mm256_set_pd(3.0,3.0,3.0,3.0)),_mm256_mul_pd(xi_267,_mm256_set_pd(3.0,3.0,3.0,3.0))),xi_108); + const __m256d xi_110 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_260,_mm256_set_pd(3.0,3.0,3.0,3.0)),_mm256_mul_pd(xi_264,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),_mm256_mul_pd(xi_270,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),_mm256_mul_pd(xi_278,_mm256_set_pd(3.0,3.0,3.0,3.0))),_mm256_mul_pd(xi_279,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),_mm256_mul_pd(xi_280,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),xi_109),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_111 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_264,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_270,_mm256_set_pd(2.0,2.0,2.0,2.0))),_mm256_mul_pd(xi_279,_mm256_set_pd(2.0,2.0,2.0,2.0))),_mm256_mul_pd(xi_280,_mm256_set_pd(2.0,2.0,2.0,2.0))); + const __m256d xi_112 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_266,_mm256_set_pd(5.0,5.0,5.0,5.0)),_mm256_mul_pd(xi_268,_mm256_set_pd(5.0,5.0,5.0,5.0))),xi_111); + const __m256d xi_113 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_260,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0)),_mm256_mul_pd(xi_262,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_272,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_275,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_276,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_278,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0))),xi_109),xi_112),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_116 = _mm256_mul_pd(xi_264,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_117 = _mm256_add_pd(xi_116,xi_22); + const __m256d xi_118 = _mm256_mul_pd(xi_259,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_121 = _mm256_mul_pd(xi_272,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_122 = _mm256_add_pd(_mm256_add_pd(xi_121,xi_19),xi_27); + const __m256d xi_124 = _mm256_mul_pd(xi_275,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_125 = _mm256_mul_pd(xi_272,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_126 = _mm256_add_pd(_mm256_mul_pd(xi_262,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_276,_mm256_set_pd(2.0,2.0,2.0,2.0))); + const __m256d xi_127 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_259,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0)),_mm256_mul_pd(xi_260,_mm256_set_pd(5.0,5.0,5.0,5.0))),_mm256_mul_pd(xi_261,_mm256_set_pd(-4.0,-4.0,-4.0,-4.0))),_mm256_mul_pd(xi_265,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0))),_mm256_mul_pd(xi_267,_mm256_set_pd(-4.0,-4.0,-4.0,-4.0))),_mm256_mul_pd(xi_269,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0))),_mm256_mul_pd(xi_274,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0))),_mm256_mul_pd(xi_278,_mm256_set_pd(5.0,5.0,5.0,5.0))),xi_108),xi_112),xi_124),xi_125),xi_126),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_128 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_116,xi_23),xi_260),xi_279),xi_38); + const __m256d xi_130 = _mm256_mul_pd(xi_128,_mm256_set_pd(xi_129,xi_129,xi_129,xi_129)); + const __m256d xi_135 = _mm256_add_pd(random_5_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_140 = _mm256_mul_pd(xi_269,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_141 = _mm256_mul_pd(xi_265,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_142 = _mm256_add_pd(_mm256_mul_pd(xi_259,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0)),_mm256_mul_pd(xi_274,_mm256_set_pd(2.0,2.0,2.0,2.0))); + const __m256d xi_143 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_140,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_141),xi_142),xi_25),xi_4); + const __m256d xi_145 = _mm256_mul_pd(xi_143,_mm256_set_pd(xi_144,xi_144,xi_144,xi_144)); + const __m256d xi_146 = _mm256_add_pd(random_3_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_151 = _mm256_add_pd(random_0_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_168 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_121,xi_13),xi_266),xi_275),xi_34); + const __m256d xi_169 = _mm256_mul_pd(xi_168,_mm256_set_pd(xi_129,xi_129,xi_129,xi_129)); + const __m256d xi_170 = _mm256_add_pd(random_4_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_172 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_141,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_14),xi_140),xi_142),xi_262),xi_266),xi_35); + const __m256d xi_173 = _mm256_mul_pd(xi_172,_mm256_set_pd(xi_144,xi_144,xi_144,xi_144)); + const __m256d xi_174 = _mm256_add_pd(random_4_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_179 = _mm256_add_pd(xi_270,xi_280); + const __m256d xi_180 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_117,xi_179),xi_261),xi_29); + const __m256d xi_181 = _mm256_mul_pd(xi_180,_mm256_set_pd(xi_129,xi_129,xi_129,xi_129)); + const __m256d xi_184 = _mm256_add_pd(random_5_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_186 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_124,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_125,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_126),xi_30),xi_6); + const __m256d xi_187 = _mm256_mul_pd(xi_186,_mm256_set_pd(xi_144,xi_144,xi_144,xi_144)); + const __m256d xi_188 = _mm256_add_pd(random_3_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5)); + const __m256d xi_195 = _mm256_mul_pd(xi_127,_mm256_set_pd(0.0138888888888889,0.0138888888888889,0.0138888888888889,0.0138888888888889)); + const __m256d xi_216 = _mm256_mul_pd(xi_113,_mm256_set_pd(-0.00714285714285714,-0.00714285714285714,-0.00714285714285714,-0.00714285714285714)); + const __m256d xi_218 = _mm256_mul_pd(xi_110,_mm256_set_pd(0.025,0.025,0.025,0.025)); + const __m256d xi_223 = _mm256_mul_pd(xi_186,_mm256_set_pd(xi_222,xi_222,xi_222,xi_222)); + const __m256d xi_225 = _mm256_mul_pd(xi_180,_mm256_set_pd(xi_224,xi_224,xi_224,xi_224)); + const __m256d xi_234 = _mm256_mul_pd(xi_143,_mm256_set_pd(xi_222,xi_222,xi_222,xi_222)); + const __m256d xi_235 = _mm256_mul_pd(xi_128,_mm256_set_pd(xi_224,xi_224,xi_224,xi_224)); + const __m256d xi_243 = _mm256_mul_pd(xi_113,_mm256_set_pd(0.0178571428571429,0.0178571428571429,0.0178571428571429,0.0178571428571429)); + const __m256d xi_249 = _mm256_mul_pd(xi_168,_mm256_set_pd(xi_224,xi_224,xi_224,xi_224)); + const __m256d xi_250 = _mm256_mul_pd(xi_172,_mm256_set_pd(xi_222,xi_222,xi_222,xi_222)); + const __m256d vel0Term = xi_2; + const __m256d vel1Term = xi_5; + const __m256d vel2Term = xi_7; + const __m256d rho = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel0Term,vel1Term),vel2Term),xi_10),xi_263),xi_8),xi_9); + const __m256d xi_11 = _mm256_div_pd(_mm256_set_pd(1.0,1.0,1.0,1.0),rho); + const __m256d xi_101 = _mm256_mul_pd(rho,_mm256_set_pd(kT,kT,kT,kT)); + const __m256d xi_102 = _mm256_sqrt_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0,-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0,-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0,-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0))); + const __m256d xi_103 = _mm256_mul_pd(_mm256_mul_pd(xi_102,_mm256_add_pd(random_6_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_set_pd(3.7416573867739413,3.7416573867739413,3.7416573867739413,3.7416573867739413)); + const __m256d xi_104 = _mm256_mul_pd(_mm256_mul_pd(xi_102,_mm256_add_pd(random_7_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_set_pd(5.4772255750516612,5.4772255750516612,5.4772255750516612,5.4772255750516612)); + const __m256d xi_106 = _mm256_mul_pd(_mm256_mul_pd(_mm256_sqrt_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0,-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0,-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0,-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0))),_mm256_add_pd(random_2_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_set_pd(xi_105,xi_105,xi_105,xi_105)); + const __m256d xi_107 = _mm256_mul_pd(_mm256_mul_pd(xi_102,_mm256_add_pd(random_6_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_set_pd(8.3666002653407556,8.3666002653407556,8.3666002653407556,8.3666002653407556)); + const __m256d xi_136 = _mm256_sqrt_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0,-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0,-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0,-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0))); + const __m256d xi_137 = _mm256_mul_pd(xi_136,_mm256_set_pd(1.4142135623730951,1.4142135623730951,1.4142135623730951,1.4142135623730951)); + const __m256d xi_138 = _mm256_mul_pd(xi_137,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_139 = _mm256_mul_pd(xi_135,xi_138); + const __m256d xi_147 = _mm256_mul_pd(xi_136,_mm256_set_pd(xi_105,xi_105,xi_105,xi_105)); + const __m256d xi_148 = _mm256_mul_pd(xi_147,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_149 = _mm256_mul_pd(xi_146,xi_148); + const __m256d xi_150 = _mm256_add_pd(_mm256_mul_pd(xi_145,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_149,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))); + const __m256d xi_152 = _mm256_sqrt_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0,-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0,-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0,-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0))); + const __m256d xi_153 = _mm256_mul_pd(xi_152,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_154 = _mm256_mul_pd(xi_151,xi_153); + const __m256d xi_158 = _mm256_add_pd(_mm256_mul_pd(xi_103,_mm256_set_pd(-0.119047619047619,-0.119047619047619,-0.119047619047619,-0.119047619047619)),_mm256_mul_pd(xi_127,_mm256_set_pd(-0.0198412698412698,-0.0198412698412698,-0.0198412698412698,-0.0198412698412698))); + const __m256d xi_160 = _mm256_mul_pd(_mm256_mul_pd(xi_152,_mm256_add_pd(random_0_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_set_pd(1.7320508075688772,1.7320508075688772,1.7320508075688772,1.7320508075688772)); + const __m256d xi_164 = _mm256_add_pd(xi_145,xi_149); + const __m256d xi_171 = _mm256_mul_pd(xi_138,xi_170); + const __m256d xi_175 = _mm256_mul_pd(xi_148,xi_174); + const __m256d xi_176 = _mm256_add_pd(xi_173,xi_175); + const __m256d xi_178 = _mm256_add_pd(_mm256_mul_pd(xi_173,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_175,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))); + const __m256d xi_185 = _mm256_mul_pd(xi_138,xi_184); + const __m256d xi_189 = _mm256_mul_pd(xi_148,xi_188); + const __m256d xi_190 = _mm256_add_pd(_mm256_mul_pd(xi_187,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_189,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))); + const __m256d xi_192 = _mm256_add_pd(xi_187,xi_189); + const __m256d xi_193 = _mm256_mul_pd(_mm256_mul_pd(xi_151,xi_152),_mm256_set_pd(0.25,0.25,0.25,0.25)); + const __m256d xi_196 = _mm256_mul_pd(xi_103,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_206 = _mm256_mul_pd(xi_153,_mm256_add_pd(random_1_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))); + const __m256d xi_215 = _mm256_mul_pd(xi_153,_mm256_add_pd(random_2_0,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))); + const __m256d xi_219 = _mm256_mul_pd(xi_107,_mm256_set_pd(-0.0142857142857143,-0.0142857142857143,-0.0142857142857143,-0.0142857142857143)); + const __m256d xi_220 = _mm256_mul_pd(xi_104,_mm256_set_pd(0.05,0.05,0.05,0.05)); + const __m256d xi_226 = _mm256_mul_pd(xi_147,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_227 = _mm256_mul_pd(xi_188,xi_226); + const __m256d xi_228 = _mm256_mul_pd(xi_137,_mm256_set_pd(0.25,0.25,0.25,0.25)); + const __m256d xi_229 = _mm256_mul_pd(xi_184,xi_228); + const __m256d xi_231 = _mm256_add_pd(_mm256_mul_pd(xi_103,_mm256_set_pd(-0.0238095238095238,-0.0238095238095238,-0.0238095238095238,-0.0238095238095238)),_mm256_mul_pd(xi_127,_mm256_set_pd(-0.00396825396825397,-0.00396825396825397,-0.00396825396825397,-0.00396825396825397))); + const __m256d xi_236 = _mm256_mul_pd(xi_146,xi_226); + const __m256d xi_237 = _mm256_mul_pd(xi_135,xi_228); + const __m256d xi_241 = _mm256_mul_pd(xi_193,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_244 = _mm256_mul_pd(xi_107,_mm256_set_pd(0.0357142857142857,0.0357142857142857,0.0357142857142857,0.0357142857142857)); + const __m256d xi_246 = _mm256_mul_pd(xi_153,_mm256_add_pd(random_1_1,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))); + const __m256d xi_251 = _mm256_mul_pd(xi_170,xi_228); + const __m256d xi_252 = _mm256_mul_pd(xi_174,xi_226); + const __m256d u_0 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(vel0Term,xi_14),xi_19)); + const __m256d xi_41 = _mm256_mul_pd(u_0,xi_277); + const __m256d xi_42 = _mm256_mul_pd(xi_41,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_48 = _mm256_mul_pd(xi_42,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_114 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_0,u_0))); + const __m256d xi_165 = _mm256_mul_pd(rho,u_0); + const __m256d xi_166 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel0Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_165),xi_32); + const __m256d xi_167 = _mm256_mul_pd(xi_166,_mm256_set_pd(xi_133,xi_133,xi_133,xi_133)); + const __m256d xi_202 = _mm256_mul_pd(xi_166,_mm256_set_pd(xi_197,xi_197,xi_197,xi_197)); + const __m256d u_1 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel1Term,xi_17),xi_21),xi_25)); + const __m256d xi_43 = _mm256_mul_pd(u_1,xi_271); + const __m256d xi_44 = _mm256_mul_pd(xi_43,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_49 = _mm256_mul_pd(xi_44,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_71 = _mm256_mul_pd(u_1,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_74 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_0,xi_72),_mm256_mul_pd(xi_277,xi_71)),_mm256_set_pd(xi_73,xi_73,xi_73,xi_73)); + const __m256d xi_75 = _mm256_mul_pd(xi_74,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_119 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_1,u_1))); + const __m256d xi_120 = _mm256_add_pd(_mm256_add_pd(xi_118,xi_119),xi_20); + const __m256d xi_131 = _mm256_mul_pd(rho,u_1); + const __m256d xi_132 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel1Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_118),xi_131),xi_265),xi_37); + const __m256d xi_134 = _mm256_mul_pd(xi_132,_mm256_set_pd(xi_133,xi_133,xi_133,xi_133)); + const __m256d xi_198 = _mm256_mul_pd(xi_132,_mm256_set_pd(xi_197,xi_197,xi_197,xi_197)); + const __m256d u_2 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel2Term,xi_27),xi_272),xi_30)); + const __m256d xi_45 = _mm256_mul_pd(u_2,xi_273); + const __m256d xi_46 = _mm256_mul_pd(xi_45,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_47 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(xi_42,xi_44),xi_46),_mm256_set_pd(xi_40 + 2.0,xi_40 + 2.0,xi_40 + 2.0,xi_40 + 2.0)); + const __m256d xi_50 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_45,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_48),xi_49); + const __m256d xi_54 = _mm256_mul_pd(xi_46,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_55 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_43,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_48),xi_54); + const __m256d xi_56 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_41,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_49),xi_54); + const __m256d xi_59 = _mm256_mul_pd(xi_50,_mm256_set_pd(xi_58,xi_58,xi_58,xi_58)); + const __m256d xi_60 = _mm256_mul_pd(xi_59,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_61 = _mm256_mul_pd(xi_56,_mm256_set_pd(xi_58,xi_58,xi_58,xi_58)); + const __m256d xi_62 = _mm256_mul_pd(xi_61,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_64 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_55,_mm256_set_pd(xi_63,xi_63,xi_63,xi_63)),xi_60),xi_62); + const __m256d xi_66 = _mm256_mul_pd(xi_55,_mm256_set_pd(xi_58,xi_58,xi_58,xi_58)); + const __m256d xi_67 = _mm256_mul_pd(xi_66,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_68 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_56,_mm256_set_pd(xi_63,xi_63,xi_63,xi_63)),xi_60),xi_67); + const __m256d xi_70 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_50,_mm256_set_pd(xi_63,xi_63,xi_63,xi_63)),xi_62),xi_67); + const __m256d xi_77 = _mm256_add_pd(_mm256_mul_pd(xi_76,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_61); + const __m256d xi_79 = _mm256_mul_pd(_mm256_mul_pd(xi_50,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_78,xi_78,xi_78,xi_78)); + const __m256d xi_81 = _mm256_mul_pd(xi_47,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_82 = _mm256_add_pd(xi_66,xi_81); + const __m256d xi_83 = _mm256_add_pd(xi_80,xi_82); + const __m256d xi_84 = _mm256_add_pd(xi_79,xi_83); + const __m256d xi_85 = _mm256_add_pd(xi_61,xi_76); + const __m256d xi_86 = _mm256_add_pd(_mm256_mul_pd(xi_80,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_82); + const __m256d xi_87 = _mm256_add_pd(xi_79,xi_86); + const __m256d xi_88 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_2,xi_72),_mm256_mul_pd(xi_273,xi_71)),_mm256_set_pd(xi_73,xi_73,xi_73,xi_73)); + const __m256d xi_89 = _mm256_mul_pd(_mm256_mul_pd(xi_56,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_78,xi_78,xi_78,xi_78)); + const __m256d xi_91 = _mm256_add_pd(xi_59,xi_90); + const __m256d xi_92 = _mm256_add_pd(xi_89,xi_91); + const __m256d xi_93 = _mm256_mul_pd(xi_88,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_94 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_mul_pd(u_0,xi_273),_mm256_set_pd(0.5,0.5,0.5,0.5)),_mm256_mul_pd(_mm256_mul_pd(u_2,xi_277),_mm256_set_pd(0.5,0.5,0.5,0.5))),_mm256_set_pd(xi_73,xi_73,xi_73,xi_73)); + const __m256d xi_95 = _mm256_mul_pd(xi_94,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_96 = _mm256_mul_pd(_mm256_mul_pd(xi_55,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_78,xi_78,xi_78,xi_78)); + const __m256d xi_97 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_91),xi_96); + const __m256d xi_98 = _mm256_add_pd(_mm256_mul_pd(xi_90,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_59); + const __m256d xi_99 = _mm256_add_pd(xi_89,xi_98); + const __m256d xi_100 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_96),xi_98); + const __m256d xi_115 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_2,u_2))); + const __m256d xi_123 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_114,xi_115),xi_117),xi_120),xi_122),xi_24),xi_263),xi_28),_mm256_set_pd(omega_bulk,omega_bulk,omega_bulk,omega_bulk)); + const __m256d xi_155 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_115,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_261),xi_267); + const __m256d xi_156 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_260,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_1),xi_120),xi_155),xi_18),xi_23),xi_31),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_157 = _mm256_mul_pd(xi_156,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_159 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_114,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_119,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),_mm256_mul_pd(xi_266,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0))),_mm256_mul_pd(xi_268,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0))),xi_111),xi_118),xi_122),xi_155),xi_20),xi_260),xi_278),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_161 = _mm256_add_pd(_mm256_mul_pd(xi_159,_mm256_set_pd(-0.0416666666666667,-0.0416666666666667,-0.0416666666666667,-0.0416666666666667)),_mm256_mul_pd(xi_160,_mm256_set_pd(-0.166666666666667,-0.166666666666667,-0.166666666666667,-0.166666666666667))); + const __m256d xi_162 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(-0.1,-0.1,-0.1,-0.1)),_mm256_mul_pd(xi_110,_mm256_set_pd(-0.05,-0.05,-0.05,-0.05))),xi_161); + const __m256d xi_163 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_107,_mm256_set_pd(0.0285714285714286,0.0285714285714286,0.0285714285714286,0.0285714285714286)),_mm256_mul_pd(xi_113,_mm256_set_pd(0.0142857142857143,0.0142857142857143,0.0142857142857143,0.0142857142857143))),xi_154),xi_157),xi_158),xi_162); + const __m256d xi_177 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_107,_mm256_set_pd(-0.0714285714285714,-0.0714285714285714,-0.0714285714285714,-0.0714285714285714)),_mm256_mul_pd(xi_113,_mm256_set_pd(-0.0357142857142857,-0.0357142857142857,-0.0357142857142857,-0.0357142857142857))),_mm256_mul_pd(xi_159,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333))),_mm256_mul_pd(xi_160,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333))),xi_158); + const __m256d xi_182 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(rho,u_2),_mm256_mul_pd(vel2Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_10),xi_116),xi_121),xi_179),xi_262); + const __m256d xi_183 = _mm256_mul_pd(xi_182,_mm256_set_pd(xi_133,xi_133,xi_133,xi_133)); + const __m256d xi_191 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_103,_mm256_set_pd(0.0952380952380952,0.0952380952380952,0.0952380952380952,0.0952380952380952)),_mm256_mul_pd(xi_107,_mm256_set_pd(-0.0428571428571429,-0.0428571428571429,-0.0428571428571429,-0.0428571428571429))),_mm256_mul_pd(xi_113,_mm256_set_pd(-0.0214285714285714,-0.0214285714285714,-0.0214285714285714,-0.0214285714285714))),_mm256_mul_pd(xi_127,_mm256_set_pd(0.0158730158730159,0.0158730158730159,0.0158730158730159,0.0158730158730159))),_mm256_mul_pd(xi_154,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),_mm256_mul_pd(xi_157,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_162); + const __m256d xi_194 = _mm256_mul_pd(xi_156,_mm256_set_pd(0.0625,0.0625,0.0625,0.0625)); + const __m256d xi_199 = _mm256_add_pd(_mm256_mul_pd(xi_106,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)),_mm256_mul_pd(xi_123,_mm256_set_pd(0.0416666666666667,0.0416666666666667,0.0416666666666667,0.0416666666666667))); + const __m256d xi_200 = _mm256_add_pd(xi_198,xi_199); + const __m256d xi_201 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_164,xi_193),xi_194),xi_195),xi_196),xi_200); + const __m256d xi_203 = _mm256_add_pd(_mm256_mul_pd(xi_159,_mm256_set_pd(0.0208333333333333,0.0208333333333333,0.0208333333333333,0.0208333333333333)),_mm256_mul_pd(xi_160,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333))); + const __m256d xi_204 = _mm256_add_pd(_mm256_mul_pd(xi_202,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_203); + const __m256d xi_205 = _mm256_add_pd(xi_178,xi_204); + const __m256d xi_211 = _mm256_add_pd(xi_202,xi_203); + const __m256d xi_212 = _mm256_add_pd(xi_176,xi_211); + const __m256d xi_213 = _mm256_add_pd(_mm256_mul_pd(xi_198,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_199); + const __m256d xi_214 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_150,xi_193),xi_194),xi_195),xi_196),xi_213); + const __m256d xi_230 = _mm256_mul_pd(xi_182,_mm256_set_pd(xi_197,xi_197,xi_197,xi_197)); + const __m256d xi_232 = _mm256_add_pd(xi_230,xi_231); + const __m256d xi_233 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_223,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_227,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_225),xi_229),xi_232); + const __m256d xi_238 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_234,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_236,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_200),xi_235),xi_237); + const __m256d xi_239 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_235,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_237,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_213),xi_234),xi_236); + const __m256d xi_242 = _mm256_mul_pd(xi_194,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_245 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_192,xi_199),xi_232),xi_241),xi_242),xi_243),xi_244); + const __m256d xi_253 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_249,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_251,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_204),xi_250),xi_252); + const __m256d xi_255 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_250,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_252,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_211),xi_249),xi_251); + const __m256d xi_256 = _mm256_add_pd(_mm256_mul_pd(xi_230,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_231); + const __m256d xi_257 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_225,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_229,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_223),xi_227),xi_256); + const __m256d xi_258 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_190,xi_199),xi_241),xi_242),xi_243),xi_244),xi_256); + const __m256d p_0 = _mm256_add_pd(xi_2,xi_32); + const __m256d p_1 = xi_33; + const __m256d xi_208 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_1,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_0,xi_131)),_mm256_set_pd(xi_207,xi_207,xi_207,xi_207)); + const __m256d xi_209 = _mm256_add_pd(_mm256_mul_pd(xi_206,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_208,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))); + const __m256d xi_210 = _mm256_add_pd(xi_206,xi_208); + const __m256d p_2 = xi_36; + const __m256d xi_247 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_2,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_165)),_mm256_set_pd(xi_207,xi_207,xi_207,xi_207)); + const __m256d xi_248 = _mm256_add_pd(_mm256_mul_pd(xi_246,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_247,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))); + const __m256d xi_254 = _mm256_add_pd(xi_246,xi_247); + const __m256d p_3 = xi_33; + const __m256d p_4 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_37),xi_5); + const __m256d p_5 = xi_39; + const __m256d xi_217 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_5,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_131)),_mm256_set_pd(xi_207,xi_207,xi_207,xi_207)); + const __m256d xi_221 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_161,xi_215),xi_216),xi_217),xi_218),xi_219),xi_220); + const __m256d xi_240 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_215,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_217,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_161),xi_216),xi_218),xi_219),xi_220); + const __m256d p_6 = xi_36; + const __m256d p_7 = xi_39; + const __m256d p_8 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_1,xi_10),xi_280),xi_3),xi_7); + const __m256d forceTerm_0 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_47,_mm256_set_pd(-1.5,-1.5,-1.5,-1.5)),_mm256_mul_pd(_mm256_mul_pd(xi_50,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_53,xi_53,xi_53,xi_53))),_mm256_mul_pd(_mm256_mul_pd(xi_55,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_53,xi_53,xi_53,xi_53))),_mm256_mul_pd(_mm256_mul_pd(xi_56,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_53,xi_53,xi_53,xi_53))); + const __m256d forceTerm_1 = _mm256_add_pd(xi_57,xi_64); + const __m256d forceTerm_2 = _mm256_add_pd(_mm256_mul_pd(xi_57,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_64); + const __m256d forceTerm_3 = _mm256_add_pd(_mm256_mul_pd(xi_65,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_68); + const __m256d forceTerm_4 = _mm256_add_pd(xi_65,xi_68); + const __m256d forceTerm_5 = _mm256_add_pd(xi_69,xi_70); + const __m256d forceTerm_6 = _mm256_add_pd(_mm256_mul_pd(xi_69,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_70); + const __m256d forceTerm_7 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_77),xi_84); + const __m256d forceTerm_8 = _mm256_add_pd(_mm256_add_pd(xi_74,xi_84),xi_85); + const __m256d forceTerm_9 = _mm256_add_pd(_mm256_add_pd(xi_74,xi_77),xi_87); + const __m256d forceTerm_10 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_85),xi_87); + const __m256d forceTerm_11 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_88),xi_92); + const __m256d forceTerm_12 = _mm256_add_pd(_mm256_add_pd(xi_86,xi_92),xi_93); + const __m256d forceTerm_13 = _mm256_add_pd(_mm256_add_pd(xi_77,xi_95),xi_97); + const __m256d forceTerm_14 = _mm256_add_pd(_mm256_add_pd(xi_85,xi_94),xi_97); + const __m256d forceTerm_15 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_93),xi_99); + const __m256d forceTerm_16 = _mm256_add_pd(_mm256_add_pd(xi_86,xi_88),xi_99); + const __m256d forceTerm_17 = _mm256_add_pd(_mm256_add_pd(xi_100,xi_77),xi_94); + const __m256d forceTerm_18 = _mm256_add_pd(_mm256_add_pd(xi_100,xi_85),xi_95); + _mm256_store_pd(&_data_pdfs_20_30_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_103,_mm256_set_pd(0.142857142857143,0.142857142857143,0.142857142857143,0.142857142857143)),_mm256_mul_pd(xi_104,_mm256_set_pd(0.2,0.2,0.2,0.2))),_mm256_mul_pd(xi_106,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),_mm256_mul_pd(xi_107,_mm256_set_pd(0.0857142857142857,0.0857142857142857,0.0857142857142857,0.0857142857142857))),_mm256_mul_pd(xi_110,_mm256_set_pd(0.1,0.1,0.1,0.1))),_mm256_mul_pd(xi_113,_mm256_set_pd(0.0428571428571429,0.0428571428571429,0.0428571428571429,0.0428571428571429))),_mm256_mul_pd(xi_123,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_mul_pd(xi_127,_mm256_set_pd(0.0238095238095238,0.0238095238095238,0.0238095238095238,0.0238095238095238))),forceTerm_0),xi_263)); + _mm256_store_pd(&_data_pdfs_20_31_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_130,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_139,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),forceTerm_1),xi_134),xi_150),xi_163),xi_260)); + _mm256_store_pd(&_data_pdfs_20_32_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_134,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_2),xi_130),xi_139),xi_163),xi_164),xi_278)); + _mm256_store_pd(&_data_pdfs_20_33_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_167,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_3),xi_169),xi_171),xi_176),xi_177),xi_268)); + _mm256_store_pd(&_data_pdfs_20_34_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_169,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_171,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),forceTerm_4),xi_167),xi_177),xi_178),xi_266)); + _mm256_store_pd(&_data_pdfs_20_35_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_181,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_185,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),forceTerm_5),xi_183),xi_190),xi_191),xi_261)); + _mm256_store_pd(&_data_pdfs_20_36_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_183,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_6),xi_181),xi_185),xi_191),xi_192),xi_267)); + _mm256_store_pd(&_data_pdfs_20_37_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_7,xi_201),xi_205),xi_209),xi_269)); + _mm256_store_pd(&_data_pdfs_20_38_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_8,xi_201),xi_210),xi_212),xi_259)); + _mm256_store_pd(&_data_pdfs_20_39_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_9,xi_205),xi_210),xi_214),xi_274)); + _mm256_store_pd(&_data_pdfs_20_310_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_10,xi_209),xi_212),xi_214),xi_265)); + _mm256_store_pd(&_data_pdfs_20_311_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_11,xi_221),xi_233),xi_238),xi_264)); + _mm256_store_pd(&_data_pdfs_20_312_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_12,xi_233),xi_239),xi_240),xi_279)); + _mm256_store_pd(&_data_pdfs_20_313_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_13,xi_245),xi_248),xi_253),xi_275)); + _mm256_store_pd(&_data_pdfs_20_314_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_14,xi_245),xi_254),xi_255),xi_272)); + _mm256_store_pd(&_data_pdfs_20_315_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_15,xi_238),xi_240),xi_257),xi_270)); + _mm256_store_pd(&_data_pdfs_20_316_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_16,xi_221),xi_239),xi_257),xi_280)); + _mm256_store_pd(&_data_pdfs_20_317_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_17,xi_253),xi_254),xi_258),xi_276)); + _mm256_store_pd(&_data_pdfs_20_318_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_18,xi_248),xi_255),xi_258),xi_262)); + } } - } - } + } +} } -} // namespace internal_kernel_collide namespace internal_kernel_stream { -static FUNC_PREFIX void -kernel_stream(double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, - int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_1, - int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3) { - for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - for (int64_t ctr_0 = 1; - ctr_0 < ((_size_pdfs_0 - 2) % (4) == 0 - ? _size_pdfs_0 - 2 - : ((int64_t)((_size_pdfs_0 - 2) / (4)) + 1) * (4)) + - 1; - ctr_0 += 4) { - _mm256_store_pd(&_data_pdfs_tmp_20_30_10[ctr_0], - _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_31_10[ctr_0], - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_32_10[ctr_0], - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_33_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_34_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_35_10[ctr_0], - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_36_10[ctr_0], - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_37_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_38_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_39_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_310_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_311_10[ctr_0], - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_312_10[ctr_0], - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_313_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_314_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_315_10[ctr_0], - _mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_316_10[ctr_0], - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_317_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_318_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])); +static FUNC_PREFIX void kernel_stream(double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3) +{ + for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + for (int64_t ctr_0 = 1; ctr_0 < ((_size_pdfs_0 - 2) % (4) == 0 ? _size_pdfs_0 - 2 : ((int64_t)((_size_pdfs_0 - 2) / (4))+1) * (4)) + 1; ctr_0 += 4) + { + _mm256_store_pd(&_data_pdfs_tmp_20_30_10[ctr_0],_mm256_load_pd(& _data_pdfs_20_30_10[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_31_10[ctr_0],_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_32_10[ctr_0],_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_33_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_34_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_35_10[ctr_0],_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_36_10[ctr_0],_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_37_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_38_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_39_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_310_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_311_10[ctr_0],_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_312_10[ctr_0],_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_313_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_314_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_315_10[ctr_0],_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_316_10[ctr_0],_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_317_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_318_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + } } - } - } + } } -} // namespace internal_kernel_stream +} + -const real_t FluctuatingMRTLatticeModelAvx::w[19] = { - 0.333333333333333, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778}; -const real_t FluctuatingMRTLatticeModelAvx::wInv[19] = { - 3.00000000000000, 18.0000000000000, 18.0000000000000, 18.0000000000000, - 18.0000000000000, 18.0000000000000, 18.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000}; +const real_t FluctuatingMRTLatticeModelAvx::w[19] = { 0.333333333333333,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778 }; +const real_t FluctuatingMRTLatticeModelAvx::wInv[19] = { 3.00000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000 }; -void FluctuatingMRTLatticeModelAvx::Sweep::streamCollide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); +void FluctuatingMRTLatticeModelAvx::Sweep::streamCollide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - auto &lm = dynamic_cast *>(pdfs) - ->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &omega_odd = lm.omega_odd_; - auto &time_step = lm.time_step_; - auto &block_offset_1 = lm.block_offset_1_; - auto &force = lm.force_; - auto &block_offset_0 = lm.block_offset_0_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &kT = lm.kT_; - auto &seed = lm.seed_; - auto &block_offset_2 = lm.block_offset_2_; - auto &omega_bulk = lm.omega_bulk_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_streamCollide::kernel_streamCollide( - _data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, - _size_force_2, _stride_force_1, _stride_force_2, _stride_force_3, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, block_offset_0, block_offset_1, - block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, - time_step); - pdfs->swapDataPointers(pdfs_tmp); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & force = lm.force_; + auto & omega_even = lm.omega_even_; + auto & omega_odd = lm.omega_odd_; + auto & block_offset_2 = lm.block_offset_2_; + auto & block_offset_0 = lm.block_offset_0_; + auto & omega_shear = lm.omega_shear_; + auto & block_offset_1 = lm.block_offset_1_; + auto & omega_bulk = lm.omega_bulk_; + auto & seed = lm.seed_; + auto & kT = lm.kT_; + auto & time_step = lm.time_step_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_streamCollide::kernel_streamCollide(_data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, _size_force_2, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, time_step); + pdfs->swapDataPointers(pdfs_tmp); + } -void FluctuatingMRTLatticeModelAvx::Sweep::collide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); +void FluctuatingMRTLatticeModelAvx::Sweep::collide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + - auto &lm = dynamic_cast *>(pdfs) - ->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); - auto &omega_odd = lm.omega_odd_; - auto &time_step = lm.time_step_; - auto &block_offset_1 = lm.block_offset_1_; - auto &force = lm.force_; - auto &block_offset_0 = lm.block_offset_0_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &kT = lm.kT_; - auto &seed = lm.seed_; - auto &block_offset_2 = lm.block_offset_2_; - auto &omega_bulk = lm.omega_bulk_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_kernel_collide::kernel_collide( - _data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, - _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_1, - _stride_pdfs_2, _stride_pdfs_3, block_offset_0, block_offset_1, - block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, - time_step); + auto & force = lm.force_; + auto & omega_even = lm.omega_even_; + auto & omega_odd = lm.omega_odd_; + auto & block_offset_2 = lm.block_offset_2_; + auto & block_offset_0 = lm.block_offset_0_; + auto & omega_shear = lm.omega_shear_; + auto & block_offset_1 = lm.block_offset_1_; + auto & omega_bulk = lm.omega_bulk_; + auto & seed = lm.seed_; + auto & kT = lm.kT_; + auto & time_step = lm.time_step_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_kernel_collide::kernel_collide(_data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, time_step); } -void FluctuatingMRTLatticeModelAvx::Sweep::stream( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); + +void FluctuatingMRTLatticeModelAvx::Sweep::stream( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_0 = - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_1 = - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_2 = - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_stream::kernel_stream( - _data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); - pdfs->swapDataPointers(pdfs_tmp); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_stream::kernel_stream(_data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); + + pdfs->swapDataPointers(pdfs_tmp); + } + } // namespace lbm } // namespace walberla + + + // Buffer Packing namespace walberla { namespace mpi { -mpi::SendBuffer & -operator<<(mpi::SendBuffer &buf, - const ::walberla::lbm::FluctuatingMRTLatticeModelAvx &lm) { - buf << lm.currentLevel; - return buf; +mpi::SendBuffer & operator<< (mpi::SendBuffer & buf, const ::walberla::lbm::FluctuatingMRTLatticeModelAvx & lm) +{ + buf << lm.currentLevel; + return buf; } -mpi::RecvBuffer & -operator>>(mpi::RecvBuffer &buf, - ::walberla::lbm::FluctuatingMRTLatticeModelAvx &lm) { - buf >> lm.currentLevel; - return buf; +mpi::RecvBuffer & operator>> (mpi::RecvBuffer & buf, ::walberla::lbm::FluctuatingMRTLatticeModelAvx & lm) +{ + buf >> lm.currentLevel; + return buf; } + } // namespace mpi } // namespace walberla -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic pop +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic pop #endif \ No newline at end of file diff --git a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.h b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.h index 139c83920b0..04fd6fc90b4 100644 --- a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.h +++ b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.h @@ -696,16 +696,101 @@ template<> struct PressureTensor { template< typename FieldPtrOrIterator > - static void get( Matrix3< real_t > & /* pressureTensor */, const FluctuatingMRTLatticeModelAvx & /* latticeModel */, const FieldPtrOrIterator & /* it */ ) + static void get( Matrix3< real_t > & pressureTensor, const FluctuatingMRTLatticeModelAvx & lm, const FieldPtrOrIterator & it ) { - WALBERLA_ABORT("Not implemented"); + const auto x = it.x(); + const auto y = it.y(); + const auto z = it.z(); + + const real_t f_0 = it[0]; + const real_t f_1 = it[1]; + const real_t f_2 = it[2]; + const real_t f_3 = it[3]; + const real_t f_4 = it[4]; + const real_t f_5 = it[5]; + const real_t f_6 = it[6]; + const real_t f_7 = it[7]; + const real_t f_8 = it[8]; + const real_t f_9 = it[9]; + const real_t f_10 = it[10]; + const real_t f_11 = it[11]; + const real_t f_12 = it[12]; + const real_t f_13 = it[13]; + const real_t f_14 = it[14]; + const real_t f_15 = it[15]; + const real_t f_16 = it[16]; + const real_t f_17 = it[17]; + const real_t f_18 = it[18]; + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } template< typename PdfField_T > - static void get( Matrix3< real_t > & /* pressureTensor */, const FluctuatingMRTLatticeModelAvx & /* latticeModel */, const PdfField_T & /* pdf */, - const cell_idx_t /* x */, const cell_idx_t /* y */, const cell_idx_t /* z */ ) + static void get( Matrix3< real_t > & pressureTensor , const FluctuatingMRTLatticeModelAvx & lm , const PdfField_T & pdf, + const cell_idx_t x, const cell_idx_t y, const cell_idx_t z ) { - WALBERLA_ABORT("Not implemented"); + const real_t & xyz0 = pdf(x,y,z,0); + const real_t f_0 = pdf.getF( &xyz0, 0); + const real_t f_1 = pdf.getF( &xyz0, 1); + const real_t f_2 = pdf.getF( &xyz0, 2); + const real_t f_3 = pdf.getF( &xyz0, 3); + const real_t f_4 = pdf.getF( &xyz0, 4); + const real_t f_5 = pdf.getF( &xyz0, 5); + const real_t f_6 = pdf.getF( &xyz0, 6); + const real_t f_7 = pdf.getF( &xyz0, 7); + const real_t f_8 = pdf.getF( &xyz0, 8); + const real_t f_9 = pdf.getF( &xyz0, 9); + const real_t f_10 = pdf.getF( &xyz0, 10); + const real_t f_11 = pdf.getF( &xyz0, 11); + const real_t f_12 = pdf.getF( &xyz0, 12); + const real_t f_13 = pdf.getF( &xyz0, 13); + const real_t f_14 = pdf.getF( &xyz0, 14); + const real_t f_15 = pdf.getF( &xyz0, 15); + const real_t f_16 = pdf.getF( &xyz0, 16); + const real_t f_17 = pdf.getF( &xyz0, 17); + const real_t f_18 = pdf.getF( &xyz0, 18); + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } }; diff --git a/src/walberla_bridge/generated_kernels/MRTLatticeModel.cpp b/src/walberla_bridge/generated_kernels/MRTLatticeModel.cpp index 565d7c2e862..ed173547ebe 100644 --- a/src/walberla_bridge/generated_kernels/MRTLatticeModel.cpp +++ b/src/walberla_bridge/generated_kernels/MRTLatticeModel.cpp @@ -18,1521 +18,1011 @@ #include -#include "MRTLatticeModel.h" #include "core/DataTypes.h" #include "core/Macros.h" #include "lbm/field/PdfField.h" #include "lbm/sweeps/Streaming.h" +#include "MRTLatticeModel.h" #ifdef _MSC_VER -#pragma warning(disable : 4458) +# pragma warning( disable : 4458 ) #endif #define FUNC_PREFIX -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#pragma GCC diagnostic ignored "-Wshadow" -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-parameter" +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wfloat-equal" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Wunused-parameter" #endif + + + using namespace std; namespace walberla { namespace lbm { namespace internal_kernel_streamCollide { -static FUNC_PREFIX void kernel_streamCollide( - double *RESTRICT const _data_force, double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, - int64_t const _size_force_1, int64_t const _size_force_2, - int64_t const _stride_force_0, int64_t const _stride_force_1, - int64_t const _stride_force_2, int64_t const _stride_force_3, - int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, - int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, - int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, - int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, - double omega_bulk, double omega_even, double omega_odd, - double omega_shear) { - const double xi_35 = -omega_shear + 2.0; - const double xi_36 = xi_35 * 0.5; - const double xi_41 = xi_35 * 0.0833333333333333; - const double xi_46 = xi_35 * 0.166666666666667; - const double xi_56 = xi_35 * 0.25; - const double xi_61 = xi_35 * 0.0416666666666667; - const double xi_106 = omega_odd * 0.25; - const double xi_112 = omega_odd * 0.0833333333333333; - const double xi_149 = omega_shear * 0.25; - const double xi_172 = omega_odd * 0.0416666666666667; - const double xi_174 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_118 = rr_0 * 0.166666666666667; - const double xi_154 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - for (int64_t ctr_0 = 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) { - const double xi_0 = - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_1 = - xi_0 + _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_2 = _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_3 = _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_4 = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_5 = _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_6 = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_8 = - -_data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_9 = - xi_8 - - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_10 = - -_data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_11 = - -_data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_12 = - -_data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -_data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - const double xi_15 = - -_data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -_data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_18 = -_data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = - -_data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -_data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_23 = -_data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_24 = xi_17 + xi_22 + xi_23 + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_40 = - 0.166666666666667 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_48 = - 0.166666666666667 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_52 = - 0.166666666666667 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_55 = - 0.5 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_59 = - 0.0833333333333333 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_63 = - 0.0833333333333333 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_73 = - 0.0833333333333333 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_84 = -_data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_85 = xi_84 + - 3.0 * _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_86 = - omega_even * - (xi_85 - 3.0 * _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_87 = - 2.0 * _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_88 = - xi_87 + - 5.0 * _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 5.0 * _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_89 = - omega_even * - (xi_85 + xi_88 - - 2.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] - - 2.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0] - - 5.0 * - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 5.0 * - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 5.0 * _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + - _stride_pdfs_0] - - 5.0 * _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - - _stride_pdfs_0]); - const double xi_92 = -_data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_93 = xi_18 + xi_92; - const double xi_94 = - -_data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_97 = - -_data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_98 = xi_11 + xi_15 + xi_21 + xi_97; - const double xi_100 = - 2.0 * - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_101 = - 2.0 * - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_102 = - 2.0 * - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 2.0 * _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_103 = - omega_even * - (xi_100 + xi_101 + xi_102 + xi_84 + xi_88 - - 4.0 * _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] - - 4.0 * _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0] - - 7.0 * - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 5.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - 5.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_104 = - xi_92 + _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_105 = xi_104 + xi_14 + xi_22 + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_107 = xi_105 * xi_106; - const double xi_108 = - 2.0 * _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_109 = - 2.0 * _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_110 = - -2.0 * - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - 2.0 * _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_111 = -xi_108 + xi_109 + xi_110 + xi_14 + xi_19 + xi_2; - const double xi_113 = xi_111 * xi_112; - const double xi_114 = -xi_113; - const double xi_116 = - xi_94 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_120 = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_124 = xi_103 * -0.0198412698412698; - const double xi_132 = - xi_97 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_133 = - xi_12 + xi_132 + xi_20 + - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_134 = xi_106 * xi_133; - const double xi_135 = xi_1 + xi_108 - xi_109 + xi_110 + xi_13; - const double xi_136 = xi_112 * xi_135; - const double xi_138 = -xi_136; - const double xi_139 = _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_140 = xi_139 + xi_23 + xi_93 + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_141 = xi_106 * xi_140; - const double xi_144 = -xi_100 - xi_101 + xi_102 + xi_24 + xi_3; - const double xi_145 = xi_112 * xi_144; - const double xi_146 = -xi_145; - const double xi_148 = xi_145; - const double xi_152 = xi_103 * 0.0138888888888889; - const double xi_168 = xi_89 * -0.00714285714285714; - const double xi_170 = xi_86 * 0.025; - const double xi_173 = xi_144 * xi_172; - const double xi_175 = xi_140 * xi_174; - const double xi_176 = xi_103 * -0.00396825396825397; - const double xi_180 = xi_111 * xi_172; - const double xi_181 = xi_105 * xi_174; - const double xi_187 = xi_89 * 0.0178571428571429; - const double xi_190 = xi_133 * xi_174; - const double xi_191 = xi_135 * xi_172; - const double vel0Term = - xi_1 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double vel1Term = - xi_2 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double vel2Term = - xi_3 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double rho = vel0Term + vel1Term + vel2Term + xi_4 + xi_5 + xi_6 + - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_7 = 1 / (rho); - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_25 = - u_0 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_26 = xi_25 * 0.333333333333333; - const double xi_32 = -xi_26; - const double xi_90 = rho * (u_0 * u_0); - const double xi_129 = rho * u_0; - const double xi_130 = - -vel0Term + xi_120 + xi_129 + xi_4 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_131 = xi_118 * xi_130; - const double xi_158 = xi_130 * xi_154; - const double u_1 = - xi_7 * - (vel1Term + xi_16 + xi_19 + xi_8 + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_27 = - u_1 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_28 = xi_27 * 0.333333333333333; - const double xi_33 = -xi_28; - const double xi_54 = u_1 * 0.5; - const double xi_57 = - xi_56 * (u_0 * xi_55 + - xi_54 * _data_force_20_30_10[_stride_force_0 * ctr_0]); - const double xi_58 = -xi_57; - const double xi_95 = rho * (u_1 * u_1); - const double xi_96 = xi_9 + xi_94 + xi_95; - const double xi_115 = rho * u_1; - const double xi_117 = - -vel1Term + xi_115 + xi_116 + xi_5 + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_119 = xi_117 * xi_118; - const double xi_150 = - xi_149 * - (u_0 * xi_115 + xi_116 + xi_8 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]); - const double xi_155 = xi_117 * xi_154; - const double xi_156 = xi_155; - const double xi_157 = xi_113 + xi_156; - const double xi_166 = -xi_155; - const double xi_167 = xi_114 + xi_166; - const double xi_182 = xi_156 - xi_180 + xi_181; - const double xi_183 = xi_166 + xi_180 - xi_181; - const double u_2 = - xi_7 * - (vel2Term + xi_21 + xi_24 + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_29 = - u_2 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_30 = xi_29 * 0.333333333333333; - const double xi_31 = (-omega_bulk + 2.0) * (xi_26 + xi_28 + xi_30); - const double xi_34 = xi_29 * 0.666666666666667 + xi_32 + xi_33; - const double xi_37 = -xi_30; - const double xi_38 = xi_27 * 0.666666666666667 + xi_32 + xi_37; - const double xi_39 = xi_25 * 0.666666666666667 + xi_33 + xi_37; - const double xi_42 = xi_34 * xi_41; - const double xi_43 = -xi_42; - const double xi_44 = xi_39 * xi_41; - const double xi_45 = -xi_44; - const double xi_47 = xi_38 * xi_46 + xi_43 + xi_45; - const double xi_49 = xi_38 * xi_41; - const double xi_50 = -xi_49; - const double xi_51 = xi_39 * xi_46 + xi_43 + xi_50; - const double xi_53 = xi_34 * xi_46 + xi_45 + xi_50; - const double xi_60 = xi_44 - xi_59; - const double xi_62 = -xi_34 * xi_61; - const double xi_64 = xi_31 * 0.125; - const double xi_65 = xi_49 + xi_64; - const double xi_66 = xi_63 + xi_65; - const double xi_67 = xi_62 + xi_66; - const double xi_68 = xi_44 + xi_59; - const double xi_69 = -xi_63 + xi_65; - const double xi_70 = xi_62 + xi_69; - const double xi_71 = - xi_56 * (u_2 * xi_55 + - xi_54 * _data_force_20_32_10[_stride_force_0 * ctr_0]); - const double xi_72 = -xi_39 * xi_61; - const double xi_74 = xi_42 + xi_73; - const double xi_75 = xi_72 + xi_74; - const double xi_76 = -xi_71; - const double xi_77 = - xi_56 * (u_0 * 0.5 * _data_force_20_32_10[_stride_force_0 * ctr_0] + - u_2 * 0.5 * _data_force_20_30_10[_stride_force_0 * ctr_0]); - const double xi_78 = -xi_77; - const double xi_79 = -xi_38 * xi_61; - const double xi_80 = xi_64 + xi_74 + xi_79; - const double xi_81 = xi_42 - xi_73; - const double xi_82 = xi_72 + xi_81; - const double xi_83 = xi_64 + xi_79 + xi_81; - const double xi_91 = rho * (u_2 * u_2); - const double xi_99 = - omega_bulk * (xi_17 + xi_22 + xi_90 + xi_91 + xi_93 + xi_96 + - xi_98 + _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]); - const double xi_121 = -xi_91 + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_122 = - omega_shear * (xi_0 + xi_120 + xi_121 + xi_16 + xi_96 - - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]); - const double xi_123 = xi_122 * 0.125; - const double xi_125 = - omega_shear * - (xi_121 + xi_87 + xi_9 + xi_90 * 2.0 + xi_94 - xi_95 + xi_98 - - 2.0 * - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 2.0 * - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_126 = xi_125 * -0.0416666666666667; - const double xi_127 = xi_126 + xi_86 * -0.05; - const double xi_128 = - xi_123 + xi_124 + xi_127 + xi_89 * 0.0142857142857143; - const double xi_137 = - xi_124 + xi_125 * 0.0833333333333333 + xi_89 * -0.0357142857142857; - const double xi_142 = - rho * u_2 - vel2Term + xi_139 + xi_6 + xi_92 + xi_97 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_143 = xi_118 * xi_142; - const double xi_147 = xi_103 * 0.0158730158730159 - xi_123 + xi_127 + - xi_89 * -0.0214285714285714; - const double xi_151 = xi_122 * 0.0625; - const double xi_153 = -xi_150 + xi_151 + xi_152; - const double xi_159 = xi_99 * 0.0416666666666667; - const double xi_160 = xi_125 * 0.0208333333333333 + xi_159; - const double xi_161 = -xi_158 + xi_160; - const double xi_162 = xi_138 + xi_161; - const double xi_163 = xi_150 + xi_151 + xi_152; - const double xi_164 = xi_158 + xi_160; - const double xi_165 = xi_136 + xi_164; - const double xi_169 = - xi_149 * (u_2 * xi_115 + xi_104 + xi_17 + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]); - const double xi_171 = xi_126 + xi_159 + xi_168 + xi_169 + xi_170; - const double xi_177 = xi_142 * xi_154; - const double xi_178 = xi_176 + xi_177; - const double xi_179 = -xi_173 + xi_175 + xi_178; - const double xi_184 = xi_126 + xi_159 + xi_168 - xi_169 + xi_170; - const double xi_185 = - xi_149 * - (u_2 * xi_129 + xi_10 + xi_132 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_186 = -xi_151; - const double xi_188 = -xi_185 + xi_186 + xi_187; - const double xi_189 = xi_148 + xi_178; - const double xi_192 = xi_161 - xi_190 + xi_191; - const double xi_193 = xi_185 + xi_186 + xi_187; - const double xi_194 = xi_164 + xi_190 - xi_191; - const double xi_195 = xi_176 - xi_177; - const double xi_196 = xi_173 - xi_175 + xi_195; - const double xi_197 = xi_146 + xi_195; - const double forceTerm_0 = - xi_31 * -1.5 - xi_34 * xi_36 - xi_36 * xi_38 - xi_36 * xi_39; - const double forceTerm_1 = xi_40 + xi_47; - const double forceTerm_2 = -xi_40 + xi_47; - const double forceTerm_3 = -xi_48 + xi_51; - const double forceTerm_4 = xi_48 + xi_51; - const double forceTerm_5 = xi_52 + xi_53; - const double forceTerm_6 = -xi_52 + xi_53; - const double forceTerm_7 = xi_58 + xi_60 + xi_67; - const double forceTerm_8 = xi_57 + xi_67 + xi_68; - const double forceTerm_9 = xi_57 + xi_60 + xi_70; - const double forceTerm_10 = xi_58 + xi_68 + xi_70; - const double forceTerm_11 = xi_66 + xi_71 + xi_75; - const double forceTerm_12 = xi_69 + xi_75 + xi_76; - const double forceTerm_13 = xi_60 + xi_78 + xi_80; - const double forceTerm_14 = xi_68 + xi_77 + xi_80; - const double forceTerm_15 = xi_66 + xi_76 + xi_82; - const double forceTerm_16 = xi_69 + xi_71 + xi_82; - const double forceTerm_17 = xi_60 + xi_77 + xi_83; - const double forceTerm_18 = xi_68 + xi_78 + xi_83; - _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_0 + xi_103 * 0.0238095238095238 + xi_86 * 0.1 + - xi_89 * 0.0428571428571429 + xi_99 * -0.5 + - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_1 - xi_107 + xi_114 + xi_119 + xi_128 + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_2 + xi_107 + xi_113 - xi_119 + xi_128 + - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_3 - xi_131 + xi_134 + xi_136 + xi_137 + - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_4 + xi_131 - xi_134 + xi_137 + xi_138 + - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_5 - xi_141 + xi_143 + xi_146 + xi_147 + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_6 + xi_141 - xi_143 + xi_147 + xi_148 + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_7 + xi_153 + xi_157 + xi_162 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_8 + xi_157 + xi_163 + xi_165 + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_9 + xi_162 + xi_163 + xi_167 + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_10 + xi_153 + xi_165 + xi_167 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_11 + xi_171 + xi_179 + xi_182 + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_12 + xi_179 + xi_183 + xi_184 + - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_13 + xi_188 + xi_189 + xi_192 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_14 + xi_189 + xi_193 + xi_194 + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_15 + xi_182 + xi_184 + xi_196 + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_16 + xi_171 + xi_183 + xi_196 + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_17 + xi_192 + xi_193 + xi_197 + - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_18 + xi_188 + xi_194 + xi_197 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void kernel_streamCollide(double * RESTRICT const _data_force, double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_0, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, double omega_bulk, double omega_even, double omega_odd, double omega_shear) +{ + const double xi_50 = -omega_shear + 2.0; + const double xi_51 = xi_50*0.5; + const double xi_56 = xi_50*0.0833333333333333; + const double xi_61 = xi_50*0.166666666666667; + const double xi_71 = xi_50*0.25; + const double xi_76 = xi_50*0.0416666666666667; + const double xi_120 = omega_odd*0.25; + const double xi_126 = omega_odd*0.0833333333333333; + const double xi_159 = omega_shear*0.25; + const double xi_181 = omega_odd*0.0416666666666667; + const double xi_183 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_131 = rr_0*0.166666666666667; + const double xi_164 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + for (int64_t ctr_0 = 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) + { + const double xi_0 = _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_1 = _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_2 = xi_0 + xi_1 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_3 = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_4 = xi_3 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + const double xi_5 = xi_4 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_6 = _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_7 = xi_6 + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_8 = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_9 = _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0] + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_10 = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + const double xi_12 = -_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_13 = -_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_16 = -_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_17 = -_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_21 = xi_20 + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_22 = -_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_23 = -_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + const double xi_24 = -_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + const double xi_29 = -_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + const double xi_30 = xi_24 + xi_28 + xi_29 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_31 = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_32 = xi_31 + xi_8 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_33 = xi_16 + xi_21 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_34 = xi_26 + _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_35 = xi_15 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_9 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_38 = xi_28 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_39 = xi_22 + xi_38 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_55 = 0.166666666666667*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_63 = 0.166666666666667*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_67 = 0.166666666666667*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_70 = 0.5*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_74 = 0.0833333333333333*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_78 = 0.0833333333333333*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_88 = 0.0833333333333333*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_99 = -_data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_100 = xi_99 + 3.0*_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_101 = omega_even*(xi_100 - 3.0*_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_102 = 2.0*_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_103 = xi_102 + 5.0*_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 5.0*_data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_104 = omega_even*(xi_100 + xi_103 - 2.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] - 2.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0] - 5.0*_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 5.0*_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 5.0*_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 5.0*_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]); + const double xi_107 = -_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_108 = xi_107 + xi_22; + const double xi_109 = -_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_112 = -_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_113 = xi_112 + xi_19 + xi_27; + const double xi_115 = 2.0*_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_116 = 2.0*_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_117 = 2.0*_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 2.0*_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_118 = omega_even*(xi_103 + xi_115 + xi_116 + xi_117 + xi_99 - 4.0*_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] - 4.0*_data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0] - 7.0*_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 7.0*_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 7.0*_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 7.0*_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 5.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + 5.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_119 = xi_107 + xi_23 + xi_38 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_121 = xi_119*xi_120; + const double xi_122 = 2.0*_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_123 = 2.0*_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_124 = -2.0*_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + 2.0*_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_125 = -xi_122 + xi_123 + xi_124 + xi_25 + xi_4; + const double xi_127 = xi_125*xi_126; + const double xi_128 = -xi_127; + const double xi_133 = xi_118*-0.0198412698412698; + const double xi_144 = xi_112 + xi_13 + xi_34 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_145 = xi_120*xi_144; + const double xi_146 = xi_122 - xi_123 + xi_124 + xi_14 + xi_35 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_147 = xi_126*xi_146; + const double xi_149 = -xi_147; + const double xi_150 = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_151 = xi_108 + xi_150 + xi_29 + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_152 = xi_120*xi_151; + const double xi_153 = -xi_115 - xi_116 + xi_117 + xi_30 + xi_6; + const double xi_154 = xi_126*xi_153; + const double xi_155 = -xi_154; + const double xi_162 = xi_118*0.0138888888888889; + const double xi_177 = xi_104*-0.00714285714285714; + const double xi_179 = xi_101*0.025; + const double xi_182 = xi_153*xi_181; + const double xi_184 = xi_151*xi_183; + const double xi_185 = xi_118*-0.00396825396825397; + const double xi_189 = xi_125*xi_181; + const double xi_190 = xi_119*xi_183; + const double xi_196 = xi_104*0.0178571428571429; + const double xi_199 = xi_144*xi_183; + const double xi_200 = xi_146*xi_181; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_8 + xi_9 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_11 = 1 / (rho); + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_40 = u_0*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_41 = xi_40*0.333333333333333; + const double xi_47 = -xi_41; + const double xi_105 = rho*(u_0*u_0); + const double xi_141 = rho*u_0; + const double xi_142 = -vel0Term + xi_141 + xi_32; + const double xi_143 = xi_131*xi_142; + const double xi_167 = xi_142*xi_164; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_42 = u_1*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_43 = xi_42*0.333333333333333; + const double xi_48 = -xi_43; + const double xi_69 = u_1*0.5; + const double xi_72 = xi_71*(u_0*xi_70 + xi_69*_data_force_20_30_10[_stride_force_0*ctr_0]); + const double xi_73 = -xi_72; + const double xi_110 = rho*(u_1*u_1); + const double xi_111 = xi_109 + xi_110 + xi_20; + const double xi_129 = rho*u_1; + const double xi_130 = -vel1Term + xi_109 + xi_129 + xi_37 + _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_132 = xi_130*xi_131; + const double xi_165 = xi_130*xi_164; + const double xi_166 = xi_127 + xi_165; + const double xi_175 = -xi_165; + const double xi_176 = xi_128 + xi_175; + const double xi_191 = xi_165 - xi_189 + xi_190; + const double xi_192 = xi_175 + xi_189 - xi_190; + const double u_2 = xi_11*(vel2Term + xi_27 + xi_30 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]); + const double xi_44 = u_2*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_45 = xi_44*0.333333333333333; + const double xi_46 = (-omega_bulk + 2.0)*(xi_41 + xi_43 + xi_45); + const double xi_49 = xi_44*0.666666666666667 + xi_47 + xi_48; + const double xi_52 = -xi_45; + const double xi_53 = xi_42*0.666666666666667 + xi_47 + xi_52; + const double xi_54 = xi_40*0.666666666666667 + xi_48 + xi_52; + const double xi_57 = xi_49*xi_56; + const double xi_58 = -xi_57; + const double xi_59 = xi_54*xi_56; + const double xi_60 = -xi_59; + const double xi_62 = xi_53*xi_61 + xi_58 + xi_60; + const double xi_64 = xi_53*xi_56; + const double xi_65 = -xi_64; + const double xi_66 = xi_54*xi_61 + xi_58 + xi_65; + const double xi_68 = xi_49*xi_61 + xi_60 + xi_65; + const double xi_75 = xi_59 - xi_74; + const double xi_77 = -xi_49*xi_76; + const double xi_79 = xi_46*0.125; + const double xi_80 = xi_64 + xi_79; + const double xi_81 = xi_78 + xi_80; + const double xi_82 = xi_77 + xi_81; + const double xi_83 = xi_59 + xi_74; + const double xi_84 = -xi_78 + xi_80; + const double xi_85 = xi_77 + xi_84; + const double xi_86 = xi_71*(u_2*xi_70 + xi_69*_data_force_20_32_10[_stride_force_0*ctr_0]); + const double xi_87 = -xi_54*xi_76; + const double xi_89 = xi_57 + xi_88; + const double xi_90 = xi_87 + xi_89; + const double xi_91 = -xi_86; + const double xi_92 = xi_71*(u_0*0.5*_data_force_20_32_10[_stride_force_0*ctr_0] + u_2*0.5*_data_force_20_30_10[_stride_force_0*ctr_0]); + const double xi_93 = -xi_92; + const double xi_94 = -xi_53*xi_76; + const double xi_95 = xi_79 + xi_89 + xi_94; + const double xi_96 = xi_57 - xi_88; + const double xi_97 = xi_87 + xi_96; + const double xi_98 = xi_79 + xi_94 + xi_96; + const double xi_106 = rho*(u_2*u_2); + const double xi_114 = omega_bulk*(xi_105 + xi_106 + xi_108 + xi_111 + xi_113 + xi_24 + xi_28 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]); + const double xi_134 = -xi_106 + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_135 = omega_shear*(xi_1 + xi_111 + xi_134 + xi_18 + xi_23 + xi_31 - _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]); + const double xi_136 = xi_135*0.125; + const double xi_137 = omega_shear*(xi_102 + xi_105*2.0 + xi_109 - xi_110 + xi_113 + xi_134 + xi_20 - 2.0*_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 2.0*_data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_138 = xi_137*-0.0416666666666667; + const double xi_139 = xi_101*-0.05 + xi_138; + const double xi_140 = xi_104*0.0142857142857143 + xi_133 + xi_136 + xi_139; + const double xi_148 = xi_104*-0.0357142857142857 + xi_133 + xi_137*0.0833333333333333; + const double xi_156 = rho*u_2 - vel2Term + xi_10 + xi_107 + xi_112 + xi_150 + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_157 = xi_131*xi_156; + const double xi_158 = xi_104*-0.0214285714285714 + xi_118*0.0158730158730159 - xi_136 + xi_139; + const double xi_161 = xi_135*0.0625; + const double xi_168 = xi_114*0.0416666666666667; + const double xi_169 = xi_137*0.0208333333333333 + xi_168; + const double xi_170 = -xi_167 + xi_169; + const double xi_171 = xi_149 + xi_170; + const double xi_173 = xi_167 + xi_169; + const double xi_174 = xi_147 + xi_173; + const double xi_186 = xi_156*xi_164; + const double xi_187 = xi_185 + xi_186; + const double xi_188 = -xi_182 + xi_184 + xi_187; + const double xi_195 = -xi_161; + const double xi_198 = xi_154 + xi_187; + const double xi_201 = xi_170 - xi_199 + xi_200; + const double xi_203 = xi_173 + xi_199 - xi_200; + const double xi_204 = xi_185 - xi_186; + const double xi_205 = xi_182 - xi_184 + xi_204; + const double xi_206 = xi_155 + xi_204; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_160 = xi_159*(-p_1 + u_0*xi_129); + const double xi_163 = -xi_160 + xi_161 + xi_162; + const double xi_172 = xi_160 + xi_161 + xi_162; + const double p_2 = xi_36; + const double xi_194 = xi_159*(-p_2 + u_2*xi_141); + const double xi_197 = -xi_194 + xi_195 + xi_196; + const double xi_202 = xi_194 + xi_195 + xi_196; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_178 = xi_159*(-p_5 + u_2*xi_129); + const double xi_180 = xi_138 + xi_168 + xi_177 + xi_178 + xi_179; + const double xi_193 = xi_138 + xi_168 + xi_177 - xi_178 + xi_179; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_3 + xi_7 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double forceTerm_0 = xi_46*-1.5 - xi_49*xi_51 - xi_51*xi_53 - xi_51*xi_54; + const double forceTerm_1 = xi_55 + xi_62; + const double forceTerm_2 = -xi_55 + xi_62; + const double forceTerm_3 = -xi_63 + xi_66; + const double forceTerm_4 = xi_63 + xi_66; + const double forceTerm_5 = xi_67 + xi_68; + const double forceTerm_6 = -xi_67 + xi_68; + const double forceTerm_7 = xi_73 + xi_75 + xi_82; + const double forceTerm_8 = xi_72 + xi_82 + xi_83; + const double forceTerm_9 = xi_72 + xi_75 + xi_85; + const double forceTerm_10 = xi_73 + xi_83 + xi_85; + const double forceTerm_11 = xi_81 + xi_86 + xi_90; + const double forceTerm_12 = xi_84 + xi_90 + xi_91; + const double forceTerm_13 = xi_75 + xi_93 + xi_95; + const double forceTerm_14 = xi_83 + xi_92 + xi_95; + const double forceTerm_15 = xi_81 + xi_91 + xi_97; + const double forceTerm_16 = xi_84 + xi_86 + xi_97; + const double forceTerm_17 = xi_75 + xi_92 + xi_98; + const double forceTerm_18 = xi_83 + xi_93 + xi_98; + _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_0 + xi_101*0.1 + xi_104*0.0428571428571429 + xi_114*-0.5 + xi_118*0.0238095238095238 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_1 - xi_121 + xi_128 + xi_132 + xi_140 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_2 + xi_121 + xi_127 - xi_132 + xi_140 + _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_3 - xi_143 + xi_145 + xi_147 + xi_148 + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_4 + xi_143 - xi_145 + xi_148 + xi_149 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_5 - xi_152 + xi_155 + xi_157 + xi_158 + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_6 + xi_152 + xi_154 - xi_157 + xi_158 + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_7 + xi_163 + xi_166 + xi_171 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_8 + xi_166 + xi_172 + xi_174 + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_9 + xi_171 + xi_172 + xi_176 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_10 + xi_163 + xi_174 + xi_176 + _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_11 + xi_180 + xi_188 + xi_191 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_12 + xi_188 + xi_192 + xi_193 + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_13 + xi_197 + xi_198 + xi_201 + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_14 + xi_198 + xi_202 + xi_203 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_15 + xi_191 + xi_193 + xi_205 + _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_16 + xi_180 + xi_192 + xi_205 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_17 + xi_201 + xi_202 + xi_206 + _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_18 + xi_197 + xi_203 + xi_206 + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_kernel_streamCollide namespace internal_kernel_collide { -static FUNC_PREFIX void -kernel_collide(double *RESTRICT const _data_force, double *RESTRICT _data_pdfs, - int64_t const _size_force_0, int64_t const _size_force_1, - int64_t const _size_force_2, int64_t const _stride_force_0, - int64_t const _stride_force_1, int64_t const _stride_force_2, - int64_t const _stride_force_3, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, double omega_bulk, - double omega_even, double omega_odd, double omega_shear) { - const double xi_35 = -omega_shear + 2.0; - const double xi_36 = xi_35 * 0.5; - const double xi_41 = xi_35 * 0.0833333333333333; - const double xi_46 = xi_35 * 0.166666666666667; - const double xi_56 = xi_35 * 0.25; - const double xi_61 = xi_35 * 0.0416666666666667; - const double xi_106 = omega_odd * 0.25; - const double xi_112 = omega_odd * 0.0833333333333333; - const double xi_149 = omega_shear * 0.25; - const double xi_172 = omega_odd * 0.0416666666666667; - const double xi_174 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_118 = rr_0 * 0.166666666666667; - const double xi_154 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) { - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_313 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_316 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_311 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_312 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_314 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_315 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_35 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_36 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_317 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_318 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) { - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_313; - double *RESTRICT _data_pdfs_20_316_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_316; - double *RESTRICT _data_pdfs_20_311_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_311; - double *RESTRICT _data_pdfs_20_310_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_20_38_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_20_312_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_312; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_20_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_314; - double *RESTRICT _data_pdfs_20_315_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_315; - double *RESTRICT _data_pdfs_20_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_35; - double *RESTRICT _data_pdfs_20_37_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_32_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_32; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_31_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_20_39_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_36; - double *RESTRICT _data_pdfs_20_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_317; - double *RESTRICT _data_pdfs_20_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_318; - for (int64_t ctr_0 = 0; ctr_0 < _size_force_0; ctr_0 += 1) { - const double xi_198 = _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_199 = _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_200 = _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0]; - const double xi_201 = _data_pdfs_20_313_10[_stride_pdfs_0 * ctr_0]; - const double xi_202 = _data_pdfs_20_316_10[_stride_pdfs_0 * ctr_0]; - const double xi_203 = _data_pdfs_20_311_10[_stride_pdfs_0 * ctr_0]; - const double xi_204 = _data_pdfs_20_310_10[_stride_pdfs_0 * ctr_0]; - const double xi_205 = _data_pdfs_20_38_10[_stride_pdfs_0 * ctr_0]; - const double xi_206 = _data_pdfs_20_312_10[_stride_pdfs_0 * ctr_0]; - const double xi_207 = _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0]; - const double xi_208 = _data_pdfs_20_314_10[_stride_pdfs_0 * ctr_0]; - const double xi_209 = _data_pdfs_20_315_10[_stride_pdfs_0 * ctr_0]; - const double xi_210 = _data_pdfs_20_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_211 = _data_pdfs_20_37_10[_stride_pdfs_0 * ctr_0]; - const double xi_212 = _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_213 = _data_pdfs_20_32_10[_stride_pdfs_0 * ctr_0]; - const double xi_214 = _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_215 = _data_pdfs_20_31_10[_stride_pdfs_0 * ctr_0]; - const double xi_216 = _data_pdfs_20_39_10[_stride_pdfs_0 * ctr_0]; - const double xi_217 = _data_pdfs_20_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_218 = _data_pdfs_20_317_10[_stride_pdfs_0 * ctr_0]; - const double xi_219 = _data_pdfs_20_318_10[_stride_pdfs_0 * ctr_0]; - const double xi_0 = xi_208 + xi_219; - const double xi_1 = xi_0 + xi_207; - const double xi_2 = xi_203 + xi_209 + xi_215; - const double xi_3 = xi_206 + xi_210; - const double xi_4 = xi_200 + xi_216; - const double xi_5 = xi_202 + xi_213; - const double xi_6 = xi_217 + xi_218; - const double xi_8 = -xi_216; - const double xi_9 = -xi_211 + xi_8; - const double xi_10 = -xi_218; - const double xi_11 = -xi_201; - const double xi_12 = -xi_200; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -xi_213; - const double xi_15 = -xi_204; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -xi_202; - const double xi_18 = -xi_206; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = -xi_219; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -xi_209; - const double xi_23 = -xi_217; - const double xi_24 = xi_17 + xi_203 + xi_22 + xi_23; - const double xi_40 = xi_198 * 0.166666666666667; - const double xi_48 = xi_199 * 0.166666666666667; - const double xi_52 = xi_214 * 0.166666666666667; - const double xi_55 = xi_198 * 0.5; - const double xi_59 = xi_199 * 0.0833333333333333; - const double xi_63 = xi_198 * 0.0833333333333333; - const double xi_73 = xi_214 * 0.0833333333333333; - const double xi_84 = -xi_212; - const double xi_85 = xi_210 * 3.0 + xi_217 * 3.0 + xi_84; - const double xi_86 = - omega_even * (xi_202 * -3.0 + xi_203 * -3.0 + xi_206 * -3.0 + - xi_209 * -3.0 + xi_213 * 3.0 + xi_215 * 3.0 + xi_85); - const double xi_87 = - xi_202 * 2.0 + xi_203 * 2.0 + xi_206 * 2.0 + xi_209 * 2.0; - const double xi_88 = xi_200 * 5.0 + xi_207 * 5.0 + xi_87; - const double xi_89 = - omega_even * - (xi_201 * -5.0 + xi_208 * -5.0 + xi_213 * -2.0 + xi_215 * -2.0 + - xi_218 * -5.0 + xi_219 * -5.0 + xi_85 + xi_88); - const double xi_92 = -xi_203; - const double xi_93 = xi_18 + xi_92; - const double xi_94 = -xi_205; - const double xi_97 = -xi_208; - const double xi_98 = xi_11 + xi_15 + xi_21 + xi_97; - const double xi_100 = xi_201 * 2.0; - const double xi_101 = xi_208 * 2.0; - const double xi_102 = xi_218 * 2.0 + xi_219 * 2.0; - const double xi_103 = - omega_even * - (xi_100 + xi_101 + xi_102 + xi_204 * -7.0 + xi_205 * -7.0 + - xi_210 * -4.0 + xi_211 * -7.0 + xi_213 * 5.0 + xi_215 * 5.0 + - xi_216 * -7.0 + xi_217 * -4.0 + xi_84 + xi_88); - const double xi_104 = xi_206 + xi_92; - const double xi_105 = xi_104 + xi_14 + xi_202 + xi_215 + xi_22; - const double xi_107 = xi_105 * xi_106; - const double xi_108 = xi_211 * 2.0; - const double xi_109 = xi_204 * 2.0; - const double xi_110 = xi_205 * -2.0 + xi_216 * 2.0; - const double xi_111 = -xi_108 + xi_109 + xi_110 + xi_14 + xi_19 + xi_2; - const double xi_113 = xi_111 * xi_112; - const double xi_114 = -xi_113; - const double xi_116 = xi_204 + xi_94; - const double xi_120 = xi_201 + xi_218; - const double xi_124 = xi_103 * -0.0198412698412698; - const double xi_132 = xi_201 + xi_97; - const double xi_133 = xi_12 + xi_132 + xi_20 + xi_207 + xi_218; - const double xi_134 = xi_106 * xi_133; - const double xi_135 = xi_1 + xi_108 - xi_109 + xi_110 + xi_13; - const double xi_136 = xi_112 * xi_135; - const double xi_138 = -xi_136; - const double xi_139 = xi_202 + xi_209; - const double xi_140 = xi_139 + xi_210 + xi_23 + xi_93; - const double xi_141 = xi_106 * xi_140; - const double xi_144 = -xi_100 - xi_101 + xi_102 + xi_24 + xi_3; - const double xi_145 = xi_112 * xi_144; - const double xi_146 = -xi_145; - const double xi_148 = xi_145; - const double xi_152 = xi_103 * 0.0138888888888889; - const double xi_168 = xi_89 * -0.00714285714285714; - const double xi_170 = xi_86 * 0.025; - const double xi_173 = xi_144 * xi_172; - const double xi_175 = xi_140 * xi_174; - const double xi_176 = xi_103 * -0.00396825396825397; - const double xi_180 = xi_111 * xi_172; - const double xi_181 = xi_105 * xi_174; - const double xi_187 = xi_89 * 0.0178571428571429; - const double xi_190 = xi_133 * xi_174; - const double xi_191 = xi_135 * xi_172; - const double vel0Term = xi_1 + xi_204 + xi_205; - const double vel1Term = xi_2 + xi_211; - const double vel2Term = xi_201 + xi_3; - const double rho = - vel0Term + vel1Term + vel2Term + xi_212 + xi_4 + xi_5 + xi_6; - const double xi_7 = 1 / (rho); - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_25 = u_0 * xi_199; - const double xi_26 = xi_25 * 0.333333333333333; - const double xi_32 = -xi_26; - const double xi_90 = rho * (u_0 * u_0); - const double xi_129 = rho * u_0; - const double xi_130 = -vel0Term + xi_120 + xi_129 + xi_211 + xi_4; - const double xi_131 = xi_118 * xi_130; - const double xi_158 = xi_130 * xi_154; - const double u_1 = xi_7 * (vel1Term + xi_16 + xi_19 + xi_205 + xi_8); - const double xi_27 = u_1 * xi_198; - const double xi_28 = xi_27 * 0.333333333333333; - const double xi_33 = -xi_28; - const double xi_54 = u_1 * 0.5; - const double xi_57 = xi_56 * (u_0 * xi_55 + xi_199 * xi_54); - const double xi_58 = -xi_57; - const double xi_95 = rho * (u_1 * u_1); - const double xi_96 = xi_9 + xi_94 + xi_95; - const double xi_115 = rho * u_1; - const double xi_117 = - -vel1Term + xi_115 + xi_116 + xi_206 + xi_216 + xi_5; - const double xi_119 = xi_117 * xi_118; - const double xi_150 = xi_149 * (u_0 * xi_115 + xi_116 + xi_211 + xi_8); - const double xi_155 = xi_117 * xi_154; - const double xi_156 = xi_155; - const double xi_157 = xi_113 + xi_156; - const double xi_166 = -xi_155; - const double xi_167 = xi_114 + xi_166; - const double xi_182 = xi_156 - xi_180 + xi_181; - const double xi_183 = xi_166 + xi_180 - xi_181; - const double u_2 = xi_7 * (vel2Term + xi_208 + xi_21 + xi_24); - const double xi_29 = u_2 * xi_214; - const double xi_30 = xi_29 * 0.333333333333333; - const double xi_31 = (-omega_bulk + 2.0) * (xi_26 + xi_28 + xi_30); - const double xi_34 = xi_29 * 0.666666666666667 + xi_32 + xi_33; - const double xi_37 = -xi_30; - const double xi_38 = xi_27 * 0.666666666666667 + xi_32 + xi_37; - const double xi_39 = xi_25 * 0.666666666666667 + xi_33 + xi_37; - const double xi_42 = xi_34 * xi_41; - const double xi_43 = -xi_42; - const double xi_44 = xi_39 * xi_41; - const double xi_45 = -xi_44; - const double xi_47 = xi_38 * xi_46 + xi_43 + xi_45; - const double xi_49 = xi_38 * xi_41; - const double xi_50 = -xi_49; - const double xi_51 = xi_39 * xi_46 + xi_43 + xi_50; - const double xi_53 = xi_34 * xi_46 + xi_45 + xi_50; - const double xi_60 = xi_44 - xi_59; - const double xi_62 = -xi_34 * xi_61; - const double xi_64 = xi_31 * 0.125; - const double xi_65 = xi_49 + xi_64; - const double xi_66 = xi_63 + xi_65; - const double xi_67 = xi_62 + xi_66; - const double xi_68 = xi_44 + xi_59; - const double xi_69 = -xi_63 + xi_65; - const double xi_70 = xi_62 + xi_69; - const double xi_71 = xi_56 * (u_2 * xi_55 + xi_214 * xi_54); - const double xi_72 = -xi_39 * xi_61; - const double xi_74 = xi_42 + xi_73; - const double xi_75 = xi_72 + xi_74; - const double xi_76 = -xi_71; - const double xi_77 = xi_56 * (u_0 * xi_214 * 0.5 + u_2 * xi_199 * 0.5); - const double xi_78 = -xi_77; - const double xi_79 = -xi_38 * xi_61; - const double xi_80 = xi_64 + xi_74 + xi_79; - const double xi_81 = xi_42 - xi_73; - const double xi_82 = xi_72 + xi_81; - const double xi_83 = xi_64 + xi_79 + xi_81; - const double xi_91 = rho * (u_2 * u_2); - const double xi_99 = omega_bulk * (xi_17 + xi_212 + xi_22 + xi_90 + - xi_91 + xi_93 + xi_96 + xi_98); - const double xi_121 = xi_210 + xi_217 - xi_91; - const double xi_122 = - omega_shear * (xi_0 + xi_120 + xi_121 + xi_16 - xi_215 + xi_96); - const double xi_123 = xi_122 * 0.125; - const double xi_125 = - omega_shear * - (xi_121 + xi_200 * -2.0 + xi_207 * -2.0 + xi_213 + xi_215 + xi_87 + - xi_9 + xi_90 * 2.0 + xi_94 - xi_95 + xi_98); - const double xi_126 = xi_125 * -0.0416666666666667; - const double xi_127 = xi_126 + xi_86 * -0.05; - const double xi_128 = - xi_123 + xi_124 + xi_127 + xi_89 * 0.0142857142857143; - const double xi_137 = - xi_124 + xi_125 * 0.0833333333333333 + xi_89 * -0.0357142857142857; - const double xi_142 = - rho * u_2 - vel2Term + xi_139 + xi_219 + xi_6 + xi_92 + xi_97; - const double xi_143 = xi_118 * xi_142; - const double xi_147 = xi_103 * 0.0158730158730159 - xi_123 + xi_127 + - xi_89 * -0.0214285714285714; - const double xi_151 = xi_122 * 0.0625; - const double xi_153 = -xi_150 + xi_151 + xi_152; - const double xi_159 = xi_99 * 0.0416666666666667; - const double xi_160 = xi_125 * 0.0208333333333333 + xi_159; - const double xi_161 = -xi_158 + xi_160; - const double xi_162 = xi_138 + xi_161; - const double xi_163 = xi_150 + xi_151 + xi_152; - const double xi_164 = xi_158 + xi_160; - const double xi_165 = xi_136 + xi_164; - const double xi_169 = xi_149 * (u_2 * xi_115 + xi_104 + xi_17 + xi_209); - const double xi_171 = xi_126 + xi_159 + xi_168 + xi_169 + xi_170; - const double xi_177 = xi_142 * xi_154; - const double xi_178 = xi_176 + xi_177; - const double xi_179 = -xi_173 + xi_175 + xi_178; - const double xi_184 = xi_126 + xi_159 + xi_168 - xi_169 + xi_170; - const double xi_185 = xi_149 * (u_2 * xi_129 + xi_10 + xi_132 + xi_219); - const double xi_186 = -xi_151; - const double xi_188 = -xi_185 + xi_186 + xi_187; - const double xi_189 = xi_148 + xi_178; - const double xi_192 = xi_161 - xi_190 + xi_191; - const double xi_193 = xi_185 + xi_186 + xi_187; - const double xi_194 = xi_164 + xi_190 - xi_191; - const double xi_195 = xi_176 - xi_177; - const double xi_196 = xi_173 - xi_175 + xi_195; - const double xi_197 = xi_146 + xi_195; - const double forceTerm_0 = - xi_31 * -1.5 - xi_34 * xi_36 - xi_36 * xi_38 - xi_36 * xi_39; - const double forceTerm_1 = xi_40 + xi_47; - const double forceTerm_2 = -xi_40 + xi_47; - const double forceTerm_3 = -xi_48 + xi_51; - const double forceTerm_4 = xi_48 + xi_51; - const double forceTerm_5 = xi_52 + xi_53; - const double forceTerm_6 = -xi_52 + xi_53; - const double forceTerm_7 = xi_58 + xi_60 + xi_67; - const double forceTerm_8 = xi_57 + xi_67 + xi_68; - const double forceTerm_9 = xi_57 + xi_60 + xi_70; - const double forceTerm_10 = xi_58 + xi_68 + xi_70; - const double forceTerm_11 = xi_66 + xi_71 + xi_75; - const double forceTerm_12 = xi_69 + xi_75 + xi_76; - const double forceTerm_13 = xi_60 + xi_78 + xi_80; - const double forceTerm_14 = xi_68 + xi_77 + xi_80; - const double forceTerm_15 = xi_66 + xi_76 + xi_82; - const double forceTerm_16 = xi_69 + xi_71 + xi_82; - const double forceTerm_17 = xi_60 + xi_77 + xi_83; - const double forceTerm_18 = xi_68 + xi_78 + xi_83; - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0] = - forceTerm_0 + xi_103 * 0.0238095238095238 + xi_212 + xi_86 * 0.1 + - xi_89 * 0.0428571428571429 + xi_99 * -0.5; - _data_pdfs_20_31_10[_stride_pdfs_0 * ctr_0] = - forceTerm_1 - xi_107 + xi_114 + xi_119 + xi_128 + xi_215; - _data_pdfs_20_32_10[_stride_pdfs_0 * ctr_0] = - forceTerm_2 + xi_107 + xi_113 - xi_119 + xi_128 + xi_213; - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0] = - forceTerm_3 - xi_131 + xi_134 + xi_136 + xi_137 + xi_200; - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0] = - forceTerm_4 + xi_131 - xi_134 + xi_137 + xi_138 + xi_207; - _data_pdfs_20_35_10[_stride_pdfs_0 * ctr_0] = - forceTerm_5 - xi_141 + xi_143 + xi_146 + xi_147 + xi_210; - _data_pdfs_20_36_10[_stride_pdfs_0 * ctr_0] = - forceTerm_6 + xi_141 - xi_143 + xi_147 + xi_148 + xi_217; - _data_pdfs_20_37_10[_stride_pdfs_0 * ctr_0] = - forceTerm_7 + xi_153 + xi_157 + xi_162 + xi_211; - _data_pdfs_20_38_10[_stride_pdfs_0 * ctr_0] = - forceTerm_8 + xi_157 + xi_163 + xi_165 + xi_205; - _data_pdfs_20_39_10[_stride_pdfs_0 * ctr_0] = - forceTerm_9 + xi_162 + xi_163 + xi_167 + xi_216; - _data_pdfs_20_310_10[_stride_pdfs_0 * ctr_0] = - forceTerm_10 + xi_153 + xi_165 + xi_167 + xi_204; - _data_pdfs_20_311_10[_stride_pdfs_0 * ctr_0] = - forceTerm_11 + xi_171 + xi_179 + xi_182 + xi_203; - _data_pdfs_20_312_10[_stride_pdfs_0 * ctr_0] = - forceTerm_12 + xi_179 + xi_183 + xi_184 + xi_206; - _data_pdfs_20_313_10[_stride_pdfs_0 * ctr_0] = - forceTerm_13 + xi_188 + xi_189 + xi_192 + xi_201; - _data_pdfs_20_314_10[_stride_pdfs_0 * ctr_0] = - forceTerm_14 + xi_189 + xi_193 + xi_194 + xi_208; - _data_pdfs_20_315_10[_stride_pdfs_0 * ctr_0] = - forceTerm_15 + xi_182 + xi_184 + xi_196 + xi_209; - _data_pdfs_20_316_10[_stride_pdfs_0 * ctr_0] = - forceTerm_16 + xi_171 + xi_183 + xi_196 + xi_202; - _data_pdfs_20_317_10[_stride_pdfs_0 * ctr_0] = - forceTerm_17 + xi_192 + xi_193 + xi_197 + xi_218; - _data_pdfs_20_318_10[_stride_pdfs_0 * ctr_0] = - forceTerm_18 + xi_188 + xi_194 + xi_197 + xi_219; +static FUNC_PREFIX void kernel_collide(double * RESTRICT const _data_force, double * RESTRICT _data_pdfs, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_0, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, double omega_bulk, double omega_even, double omega_odd, double omega_shear) +{ + const double xi_50 = -omega_shear + 2.0; + const double xi_51 = xi_50*0.5; + const double xi_56 = xi_50*0.0833333333333333; + const double xi_61 = xi_50*0.166666666666667; + const double xi_71 = xi_50*0.25; + const double xi_76 = xi_50*0.0416666666666667; + const double xi_120 = omega_odd*0.25; + const double xi_126 = omega_odd*0.0833333333333333; + const double xi_159 = omega_shear*0.25; + const double xi_181 = omega_odd*0.0416666666666667; + const double xi_183 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_131 = rr_0*0.166666666666667; + const double xi_164 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_20_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_pdfs_20_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + 16*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_38_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_20_31_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_35; + double * RESTRICT _data_pdfs_20_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_318; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_20_311_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_311; + double * RESTRICT _data_pdfs_20_310_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_20_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_36; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_37_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_20_315_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_315; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_pdfs_20_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_314; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_39_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_313; + double * RESTRICT _data_pdfs_20_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_317; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_pdfs_20_32_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_20_312_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_312; + double * RESTRICT _data_pdfs_20_316_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_316; + for (int64_t ctr_0 = 0; ctr_0 < _size_force_0; ctr_0 += 1) + { + const double xi_207 = _data_pdfs_20_38_10[_stride_pdfs_0*ctr_0]; + const double xi_208 = _data_pdfs_20_31_10[_stride_pdfs_0*ctr_0]; + const double xi_209 = _data_pdfs_20_35_10[_stride_pdfs_0*ctr_0]; + const double xi_210 = _data_pdfs_20_318_10[_stride_pdfs_0*ctr_0]; + const double xi_211 = _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_212 = _data_pdfs_20_311_10[_stride_pdfs_0*ctr_0]; + const double xi_213 = _data_pdfs_20_310_10[_stride_pdfs_0*ctr_0]; + const double xi_214 = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0]; + const double xi_215 = _data_pdfs_20_36_10[_stride_pdfs_0*ctr_0]; + const double xi_216 = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0]; + const double xi_217 = _data_pdfs_20_37_10[_stride_pdfs_0*ctr_0]; + const double xi_218 = _data_pdfs_20_315_10[_stride_pdfs_0*ctr_0]; + const double xi_219 = _data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_220 = _data_pdfs_20_314_10[_stride_pdfs_0*ctr_0]; + const double xi_221 = _data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_222 = _data_pdfs_20_39_10[_stride_pdfs_0*ctr_0]; + const double xi_223 = _data_pdfs_20_313_10[_stride_pdfs_0*ctr_0]; + const double xi_224 = _data_pdfs_20_317_10[_stride_pdfs_0*ctr_0]; + const double xi_225 = _data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_226 = _data_pdfs_20_32_10[_stride_pdfs_0*ctr_0]; + const double xi_227 = _data_pdfs_20_312_10[_stride_pdfs_0*ctr_0]; + const double xi_228 = _data_pdfs_20_316_10[_stride_pdfs_0*ctr_0]; + const double xi_0 = xi_207 + xi_213; + const double xi_1 = xi_210 + xi_220; + const double xi_2 = xi_0 + xi_1 + xi_214; + const double xi_3 = xi_212 + xi_218; + const double xi_4 = xi_208 + xi_3; + const double xi_5 = xi_217 + xi_4; + const double xi_6 = xi_209 + xi_227; + const double xi_7 = xi_223 + xi_6; + const double xi_8 = xi_216 + xi_222; + const double xi_9 = xi_226 + xi_228; + const double xi_10 = xi_215 + xi_224; + const double xi_12 = -xi_224; + const double xi_13 = -xi_216; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -xi_223; + const double xi_16 = -xi_217; + const double xi_17 = -xi_222; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -xi_213; + const double xi_21 = xi_20 + xi_207; + const double xi_22 = -xi_227; + const double xi_23 = -xi_226; + const double xi_24 = -xi_228; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -xi_210; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -xi_218; + const double xi_29 = -xi_215; + const double xi_30 = xi_212 + xi_24 + xi_28 + xi_29; + const double xi_31 = xi_223 + xi_224; + const double xi_32 = xi_217 + xi_31 + xi_8; + const double xi_33 = xi_16 + xi_21 + xi_222; + const double xi_34 = xi_224 + xi_26; + const double xi_35 = xi_15 + xi_220; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_222 + xi_227 + xi_9; + const double xi_38 = xi_228 + xi_28; + const double xi_39 = xi_212 + xi_22 + xi_38; + const double xi_55 = xi_219*0.166666666666667; + const double xi_63 = xi_225*0.166666666666667; + const double xi_67 = xi_221*0.166666666666667; + const double xi_70 = xi_219*0.5; + const double xi_74 = xi_225*0.0833333333333333; + const double xi_78 = xi_219*0.0833333333333333; + const double xi_88 = xi_221*0.0833333333333333; + const double xi_99 = -xi_211; + const double xi_100 = xi_209*3.0 + xi_215*3.0 + xi_99; + const double xi_101 = omega_even*(xi_100 + xi_208*3.0 + xi_212*-3.0 + xi_218*-3.0 + xi_226*3.0 + xi_227*-3.0 + xi_228*-3.0); + const double xi_102 = xi_212*2.0 + xi_218*2.0 + xi_227*2.0 + xi_228*2.0; + const double xi_103 = xi_102 + xi_214*5.0 + xi_216*5.0; + const double xi_104 = omega_even*(xi_100 + xi_103 + xi_208*-2.0 + xi_210*-5.0 + xi_220*-5.0 + xi_223*-5.0 + xi_224*-5.0 + xi_226*-2.0); + const double xi_107 = -xi_212; + const double xi_108 = xi_107 + xi_22; + const double xi_109 = -xi_207; + const double xi_112 = -xi_220; + const double xi_113 = xi_112 + xi_19 + xi_27; + const double xi_115 = xi_223*2.0; + const double xi_116 = xi_220*2.0; + const double xi_117 = xi_210*2.0 + xi_224*2.0; + const double xi_118 = omega_even*(xi_103 + xi_115 + xi_116 + xi_117 + xi_207*-7.0 + xi_208*5.0 + xi_209*-4.0 + xi_213*-7.0 + xi_215*-4.0 + xi_217*-7.0 + xi_222*-7.0 + xi_226*5.0 + xi_99); + const double xi_119 = xi_107 + xi_208 + xi_227 + xi_23 + xi_38; + const double xi_121 = xi_119*xi_120; + const double xi_122 = xi_217*2.0; + const double xi_123 = xi_213*2.0; + const double xi_124 = xi_207*-2.0 + xi_222*2.0; + const double xi_125 = -xi_122 + xi_123 + xi_124 + xi_25 + xi_4; + const double xi_127 = xi_125*xi_126; + const double xi_128 = -xi_127; + const double xi_133 = xi_118*-0.0198412698412698; + const double xi_144 = xi_112 + xi_13 + xi_214 + xi_223 + xi_34; + const double xi_145 = xi_120*xi_144; + const double xi_146 = xi_122 - xi_123 + xi_124 + xi_14 + xi_210 + xi_214 + xi_35; + const double xi_147 = xi_126*xi_146; + const double xi_149 = -xi_147; + const double xi_150 = xi_218 + xi_228; + const double xi_151 = xi_108 + xi_150 + xi_209 + xi_29; + const double xi_152 = xi_120*xi_151; + const double xi_153 = -xi_115 - xi_116 + xi_117 + xi_30 + xi_6; + const double xi_154 = xi_126*xi_153; + const double xi_155 = -xi_154; + const double xi_162 = xi_118*0.0138888888888889; + const double xi_177 = xi_104*-0.00714285714285714; + const double xi_179 = xi_101*0.025; + const double xi_182 = xi_153*xi_181; + const double xi_184 = xi_151*xi_183; + const double xi_185 = xi_118*-0.00396825396825397; + const double xi_189 = xi_125*xi_181; + const double xi_190 = xi_119*xi_183; + const double xi_196 = xi_104*0.0178571428571429; + const double xi_199 = xi_144*xi_183; + const double xi_200 = xi_146*xi_181; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_211 + xi_8 + xi_9; + const double xi_11 = 1 / (rho); + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_40 = u_0*xi_225; + const double xi_41 = xi_40*0.333333333333333; + const double xi_47 = -xi_41; + const double xi_105 = rho*(u_0*u_0); + const double xi_141 = rho*u_0; + const double xi_142 = -vel0Term + xi_141 + xi_32; + const double xi_143 = xi_131*xi_142; + const double xi_167 = xi_142*xi_164; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_42 = u_1*xi_219; + const double xi_43 = xi_42*0.333333333333333; + const double xi_48 = -xi_43; + const double xi_69 = u_1*0.5; + const double xi_72 = xi_71*(u_0*xi_70 + xi_225*xi_69); + const double xi_73 = -xi_72; + const double xi_110 = rho*(u_1*u_1); + const double xi_111 = xi_109 + xi_110 + xi_20; + const double xi_129 = rho*u_1; + const double xi_130 = -vel1Term + xi_109 + xi_129 + xi_213 + xi_37; + const double xi_132 = xi_130*xi_131; + const double xi_165 = xi_130*xi_164; + const double xi_166 = xi_127 + xi_165; + const double xi_175 = -xi_165; + const double xi_176 = xi_128 + xi_175; + const double xi_191 = xi_165 - xi_189 + xi_190; + const double xi_192 = xi_175 + xi_189 - xi_190; + const double u_2 = xi_11*(vel2Term + xi_220 + xi_27 + xi_30); + const double xi_44 = u_2*xi_221; + const double xi_45 = xi_44*0.333333333333333; + const double xi_46 = (-omega_bulk + 2.0)*(xi_41 + xi_43 + xi_45); + const double xi_49 = xi_44*0.666666666666667 + xi_47 + xi_48; + const double xi_52 = -xi_45; + const double xi_53 = xi_42*0.666666666666667 + xi_47 + xi_52; + const double xi_54 = xi_40*0.666666666666667 + xi_48 + xi_52; + const double xi_57 = xi_49*xi_56; + const double xi_58 = -xi_57; + const double xi_59 = xi_54*xi_56; + const double xi_60 = -xi_59; + const double xi_62 = xi_53*xi_61 + xi_58 + xi_60; + const double xi_64 = xi_53*xi_56; + const double xi_65 = -xi_64; + const double xi_66 = xi_54*xi_61 + xi_58 + xi_65; + const double xi_68 = xi_49*xi_61 + xi_60 + xi_65; + const double xi_75 = xi_59 - xi_74; + const double xi_77 = -xi_49*xi_76; + const double xi_79 = xi_46*0.125; + const double xi_80 = xi_64 + xi_79; + const double xi_81 = xi_78 + xi_80; + const double xi_82 = xi_77 + xi_81; + const double xi_83 = xi_59 + xi_74; + const double xi_84 = -xi_78 + xi_80; + const double xi_85 = xi_77 + xi_84; + const double xi_86 = xi_71*(u_2*xi_70 + xi_221*xi_69); + const double xi_87 = -xi_54*xi_76; + const double xi_89 = xi_57 + xi_88; + const double xi_90 = xi_87 + xi_89; + const double xi_91 = -xi_86; + const double xi_92 = xi_71*(u_0*xi_221*0.5 + u_2*xi_225*0.5); + const double xi_93 = -xi_92; + const double xi_94 = -xi_53*xi_76; + const double xi_95 = xi_79 + xi_89 + xi_94; + const double xi_96 = xi_57 - xi_88; + const double xi_97 = xi_87 + xi_96; + const double xi_98 = xi_79 + xi_94 + xi_96; + const double xi_106 = rho*(u_2*u_2); + const double xi_114 = omega_bulk*(xi_105 + xi_106 + xi_108 + xi_111 + xi_113 + xi_211 + xi_24 + xi_28); + const double xi_134 = -xi_106 + xi_209 + xi_215; + const double xi_135 = omega_shear*(xi_1 + xi_111 + xi_134 + xi_18 - xi_208 + xi_23 + xi_31); + const double xi_136 = xi_135*0.125; + const double xi_137 = omega_shear*(xi_102 + xi_105*2.0 + xi_109 - xi_110 + xi_113 + xi_134 + xi_20 + xi_208 + xi_214*-2.0 + xi_216*-2.0 + xi_226); + const double xi_138 = xi_137*-0.0416666666666667; + const double xi_139 = xi_101*-0.05 + xi_138; + const double xi_140 = xi_104*0.0142857142857143 + xi_133 + xi_136 + xi_139; + const double xi_148 = xi_104*-0.0357142857142857 + xi_133 + xi_137*0.0833333333333333; + const double xi_156 = rho*u_2 - vel2Term + xi_10 + xi_107 + xi_112 + xi_150 + xi_210; + const double xi_157 = xi_131*xi_156; + const double xi_158 = xi_104*-0.0214285714285714 + xi_118*0.0158730158730159 - xi_136 + xi_139; + const double xi_161 = xi_135*0.0625; + const double xi_168 = xi_114*0.0416666666666667; + const double xi_169 = xi_137*0.0208333333333333 + xi_168; + const double xi_170 = -xi_167 + xi_169; + const double xi_171 = xi_149 + xi_170; + const double xi_173 = xi_167 + xi_169; + const double xi_174 = xi_147 + xi_173; + const double xi_186 = xi_156*xi_164; + const double xi_187 = xi_185 + xi_186; + const double xi_188 = -xi_182 + xi_184 + xi_187; + const double xi_195 = -xi_161; + const double xi_198 = xi_154 + xi_187; + const double xi_201 = xi_170 - xi_199 + xi_200; + const double xi_203 = xi_173 + xi_199 - xi_200; + const double xi_204 = xi_185 - xi_186; + const double xi_205 = xi_182 - xi_184 + xi_204; + const double xi_206 = xi_155 + xi_204; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_160 = xi_159*(-p_1 + u_0*xi_129); + const double xi_163 = -xi_160 + xi_161 + xi_162; + const double xi_172 = xi_160 + xi_161 + xi_162; + const double p_2 = xi_36; + const double xi_194 = xi_159*(-p_2 + u_2*xi_141); + const double xi_197 = -xi_194 + xi_195 + xi_196; + const double xi_202 = xi_194 + xi_195 + xi_196; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_178 = xi_159*(-p_5 + u_2*xi_129); + const double xi_180 = xi_138 + xi_168 + xi_177 + xi_178 + xi_179; + const double xi_193 = xi_138 + xi_168 + xi_177 - xi_178 + xi_179; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_228 + xi_3 + xi_7; + const double forceTerm_0 = xi_46*-1.5 - xi_49*xi_51 - xi_51*xi_53 - xi_51*xi_54; + const double forceTerm_1 = xi_55 + xi_62; + const double forceTerm_2 = -xi_55 + xi_62; + const double forceTerm_3 = -xi_63 + xi_66; + const double forceTerm_4 = xi_63 + xi_66; + const double forceTerm_5 = xi_67 + xi_68; + const double forceTerm_6 = -xi_67 + xi_68; + const double forceTerm_7 = xi_73 + xi_75 + xi_82; + const double forceTerm_8 = xi_72 + xi_82 + xi_83; + const double forceTerm_9 = xi_72 + xi_75 + xi_85; + const double forceTerm_10 = xi_73 + xi_83 + xi_85; + const double forceTerm_11 = xi_81 + xi_86 + xi_90; + const double forceTerm_12 = xi_84 + xi_90 + xi_91; + const double forceTerm_13 = xi_75 + xi_93 + xi_95; + const double forceTerm_14 = xi_83 + xi_92 + xi_95; + const double forceTerm_15 = xi_81 + xi_91 + xi_97; + const double forceTerm_16 = xi_84 + xi_86 + xi_97; + const double forceTerm_17 = xi_75 + xi_92 + xi_98; + const double forceTerm_18 = xi_83 + xi_93 + xi_98; + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0] = forceTerm_0 + xi_101*0.1 + xi_104*0.0428571428571429 + xi_114*-0.5 + xi_118*0.0238095238095238 + xi_211; + _data_pdfs_20_31_10[_stride_pdfs_0*ctr_0] = forceTerm_1 - xi_121 + xi_128 + xi_132 + xi_140 + xi_208; + _data_pdfs_20_32_10[_stride_pdfs_0*ctr_0] = forceTerm_2 + xi_121 + xi_127 - xi_132 + xi_140 + xi_226; + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0] = forceTerm_3 - xi_143 + xi_145 + xi_147 + xi_148 + xi_216; + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0] = forceTerm_4 + xi_143 - xi_145 + xi_148 + xi_149 + xi_214; + _data_pdfs_20_35_10[_stride_pdfs_0*ctr_0] = forceTerm_5 - xi_152 + xi_155 + xi_157 + xi_158 + xi_209; + _data_pdfs_20_36_10[_stride_pdfs_0*ctr_0] = forceTerm_6 + xi_152 + xi_154 - xi_157 + xi_158 + xi_215; + _data_pdfs_20_37_10[_stride_pdfs_0*ctr_0] = forceTerm_7 + xi_163 + xi_166 + xi_171 + xi_217; + _data_pdfs_20_38_10[_stride_pdfs_0*ctr_0] = forceTerm_8 + xi_166 + xi_172 + xi_174 + xi_207; + _data_pdfs_20_39_10[_stride_pdfs_0*ctr_0] = forceTerm_9 + xi_171 + xi_172 + xi_176 + xi_222; + _data_pdfs_20_310_10[_stride_pdfs_0*ctr_0] = forceTerm_10 + xi_163 + xi_174 + xi_176 + xi_213; + _data_pdfs_20_311_10[_stride_pdfs_0*ctr_0] = forceTerm_11 + xi_180 + xi_188 + xi_191 + xi_212; + _data_pdfs_20_312_10[_stride_pdfs_0*ctr_0] = forceTerm_12 + xi_188 + xi_192 + xi_193 + xi_227; + _data_pdfs_20_313_10[_stride_pdfs_0*ctr_0] = forceTerm_13 + xi_197 + xi_198 + xi_201 + xi_223; + _data_pdfs_20_314_10[_stride_pdfs_0*ctr_0] = forceTerm_14 + xi_198 + xi_202 + xi_203 + xi_220; + _data_pdfs_20_315_10[_stride_pdfs_0*ctr_0] = forceTerm_15 + xi_191 + xi_193 + xi_205 + xi_218; + _data_pdfs_20_316_10[_stride_pdfs_0*ctr_0] = forceTerm_16 + xi_180 + xi_192 + xi_205 + xi_228; + _data_pdfs_20_317_10[_stride_pdfs_0*ctr_0] = forceTerm_17 + xi_201 + xi_202 + xi_206 + xi_224; + _data_pdfs_20_318_10[_stride_pdfs_0*ctr_0] = forceTerm_18 + xi_197 + xi_203 + xi_206 + xi_210; + } } - } - } + } +} } -} // namespace internal_kernel_collide namespace internal_kernel_stream { -static FUNC_PREFIX void kernel_stream( - double *RESTRICT const _data_pdfs, double *RESTRICT _data_pdfs_tmp, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, - int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3) { - for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - for (int64_t ctr_0 = 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) { - _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void kernel_stream(double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3) +{ + for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + for (int64_t ctr_0 = 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) + { + _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_kernel_stream -const real_t MRTLatticeModel::w[19] = { - 0.333333333333333, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778}; -const real_t MRTLatticeModel::wInv[19] = { - 3.00000000000000, 18.0000000000000, 18.0000000000000, 18.0000000000000, - 18.0000000000000, 18.0000000000000, 18.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000}; -void MRTLatticeModel::Sweep::streamCollide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); +const real_t MRTLatticeModel::w[19] = { 0.333333333333333,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778 }; +const real_t MRTLatticeModel::wInv[19] = { 3.00000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000 }; + +void MRTLatticeModel::Sweep::streamCollide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - auto &lm = - dynamic_cast *>(pdfs)->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &force = lm.force_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &omega_odd = lm.omega_odd_; - auto &omega_bulk = lm.omega_bulk_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_force_0 = int64_t(force->xStride()); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_streamCollide::kernel_streamCollide( - _data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, - _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, - _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, - _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, omega_bulk, omega_even, omega_odd, - omega_shear); - pdfs->swapDataPointers(pdfs_tmp); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & force = lm.force_; + auto & omega_even = lm.omega_even_; + auto & omega_odd = lm.omega_odd_; + auto & omega_shear = lm.omega_shear_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_force_0 = int64_t(force->xStride()); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_streamCollide::kernel_streamCollide(_data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, omega_bulk, omega_even, omega_odd, omega_shear); + pdfs->swapDataPointers(pdfs_tmp); + } -void MRTLatticeModel::Sweep::collide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); +void MRTLatticeModel::Sweep::collide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); - auto &lm = - dynamic_cast *>(pdfs)->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &force = lm.force_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &omega_odd = lm.omega_odd_; - auto &omega_bulk = lm.omega_bulk_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - const int64_t _stride_force_0 = int64_t(force->xStride()); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_kernel_collide::kernel_collide( - _data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, - _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, - omega_bulk, omega_even, omega_odd, omega_shear); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & force = lm.force_; + auto & omega_even = lm.omega_even_; + auto & omega_odd = lm.omega_odd_; + auto & omega_shear = lm.omega_shear_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + const int64_t _stride_force_0 = int64_t(force->xStride()); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_kernel_collide::kernel_collide(_data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, omega_bulk, omega_even, omega_odd, omega_shear); } -void MRTLatticeModel::Sweep::stream(IBlock *block, - const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); + +void MRTLatticeModel::Sweep::stream( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_0 = - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_1 = - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_2 = - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_stream::kernel_stream( - _data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, - _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, - _stride_pdfs_tmp_3); - pdfs->swapDataPointers(pdfs_tmp); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_stream::kernel_stream(_data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); + + pdfs->swapDataPointers(pdfs_tmp); + } + } // namespace lbm } // namespace walberla + + + // Buffer Packing namespace walberla { namespace mpi { -mpi::SendBuffer &operator<<(mpi::SendBuffer &buf, - const ::walberla::lbm::MRTLatticeModel &lm) { - buf << lm.currentLevel; - return buf; +mpi::SendBuffer & operator<< (mpi::SendBuffer & buf, const ::walberla::lbm::MRTLatticeModel & lm) +{ + buf << lm.currentLevel; + return buf; } -mpi::RecvBuffer &operator>>(mpi::RecvBuffer &buf, - ::walberla::lbm::MRTLatticeModel &lm) { - buf >> lm.currentLevel; - return buf; +mpi::RecvBuffer & operator>> (mpi::RecvBuffer & buf, ::walberla::lbm::MRTLatticeModel & lm) +{ + buf >> lm.currentLevel; + return buf; } + } // namespace mpi } // namespace walberla -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic pop +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic pop #endif \ No newline at end of file diff --git a/src/walberla_bridge/generated_kernels/MRTLatticeModel.h b/src/walberla_bridge/generated_kernels/MRTLatticeModel.h index 3b3221b0e11..1ec39c7b520 100644 --- a/src/walberla_bridge/generated_kernels/MRTLatticeModel.h +++ b/src/walberla_bridge/generated_kernels/MRTLatticeModel.h @@ -687,16 +687,101 @@ template<> struct PressureTensor { template< typename FieldPtrOrIterator > - static void get( Matrix3< real_t > & /* pressureTensor */, const MRTLatticeModel & /* latticeModel */, const FieldPtrOrIterator & /* it */ ) + static void get( Matrix3< real_t > & pressureTensor, const MRTLatticeModel & lm, const FieldPtrOrIterator & it ) { - WALBERLA_ABORT("Not implemented"); + const auto x = it.x(); + const auto y = it.y(); + const auto z = it.z(); + + const real_t f_0 = it[0]; + const real_t f_1 = it[1]; + const real_t f_2 = it[2]; + const real_t f_3 = it[3]; + const real_t f_4 = it[4]; + const real_t f_5 = it[5]; + const real_t f_6 = it[6]; + const real_t f_7 = it[7]; + const real_t f_8 = it[8]; + const real_t f_9 = it[9]; + const real_t f_10 = it[10]; + const real_t f_11 = it[11]; + const real_t f_12 = it[12]; + const real_t f_13 = it[13]; + const real_t f_14 = it[14]; + const real_t f_15 = it[15]; + const real_t f_16 = it[16]; + const real_t f_17 = it[17]; + const real_t f_18 = it[18]; + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } template< typename PdfField_T > - static void get( Matrix3< real_t > & /* pressureTensor */, const MRTLatticeModel & /* latticeModel */, const PdfField_T & /* pdf */, - const cell_idx_t /* x */, const cell_idx_t /* y */, const cell_idx_t /* z */ ) + static void get( Matrix3< real_t > & pressureTensor , const MRTLatticeModel & lm , const PdfField_T & pdf, + const cell_idx_t x, const cell_idx_t y, const cell_idx_t z ) { - WALBERLA_ABORT("Not implemented"); + const real_t & xyz0 = pdf(x,y,z,0); + const real_t f_0 = pdf.getF( &xyz0, 0); + const real_t f_1 = pdf.getF( &xyz0, 1); + const real_t f_2 = pdf.getF( &xyz0, 2); + const real_t f_3 = pdf.getF( &xyz0, 3); + const real_t f_4 = pdf.getF( &xyz0, 4); + const real_t f_5 = pdf.getF( &xyz0, 5); + const real_t f_6 = pdf.getF( &xyz0, 6); + const real_t f_7 = pdf.getF( &xyz0, 7); + const real_t f_8 = pdf.getF( &xyz0, 8); + const real_t f_9 = pdf.getF( &xyz0, 9); + const real_t f_10 = pdf.getF( &xyz0, 10); + const real_t f_11 = pdf.getF( &xyz0, 11); + const real_t f_12 = pdf.getF( &xyz0, 12); + const real_t f_13 = pdf.getF( &xyz0, 13); + const real_t f_14 = pdf.getF( &xyz0, 14); + const real_t f_15 = pdf.getF( &xyz0, 15); + const real_t f_16 = pdf.getF( &xyz0, 16); + const real_t f_17 = pdf.getF( &xyz0, 17); + const real_t f_18 = pdf.getF( &xyz0, 18); + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } }; diff --git a/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.cpp b/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.cpp index 24808dac536..7dbb23cd203 100644 --- a/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.cpp +++ b/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.cpp @@ -18,2877 +18,1006 @@ #include -#include "MRTLatticeModelAvx.h" #include "core/DataTypes.h" #include "core/Macros.h" #include "lbm/field/PdfField.h" #include "lbm/sweeps/Streaming.h" +#include "MRTLatticeModelAvx.h" #ifdef _MSC_VER -#pragma warning(disable : 4458) +# pragma warning( disable : 4458 ) #endif #define FUNC_PREFIX -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#pragma GCC diagnostic ignored "-Wshadow" -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-parameter" +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wfloat-equal" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Wunused-parameter" #endif + #include + + using namespace std; namespace walberla { namespace lbm { namespace internal_kernel_streamCollide { -static FUNC_PREFIX void kernel_streamCollide( - double *RESTRICT const _data_force, double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, - int64_t const _size_force_1, int64_t const _size_force_2, - int64_t const _stride_force_1, int64_t const _stride_force_2, - int64_t const _stride_force_3, int64_t const _stride_pdfs_1, - int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, - int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3, double omega_bulk, double omega_even, - double omega_odd, double omega_shear) { - const double xi_35 = -omega_shear + 2.0; - const double xi_36 = xi_35 * 0.5; - const double xi_41 = xi_35 * 0.0833333333333333; - const double xi_46 = xi_35 * 0.166666666666667; - const double xi_56 = xi_35 * 0.25; - const double xi_61 = xi_35 * 0.0416666666666667; - const double xi_106 = omega_odd * 0.25; - const double xi_112 = omega_odd * 0.0833333333333333; - const double xi_149 = omega_shear * 0.25; - const double xi_172 = omega_odd * 0.0416666666666667; - const double xi_174 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_118 = rr_0 * 0.166666666666667; - const double xi_154 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - for (int64_t ctr_0 = 1; - ctr_0 < ((_size_force_0 - 2) % (4) == 0 - ? _size_force_0 - 2 - : ((int64_t)((_size_force_0 - 2) / (4)) + 1) * (4)) + - 1; - ctr_0 += 4) { - const __m256d xi_0 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1]), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - const __m256d xi_1 = _mm256_add_pd( - xi_0, _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1])); - const __m256d xi_2 = _mm256_add_pd( - _mm256_add_pd(_mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0]), - _mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0])), - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - const __m256d xi_3 = - _mm256_add_pd(_mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0]), - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0])); - const __m256d xi_4 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1]), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])); - const __m256d xi_5 = - _mm256_add_pd(_mm256_load_pd(&_data_pdfs_20_32_11[ctr_0]), - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_6 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1]), - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0])); - const __m256d xi_8 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])); - const __m256d xi_9 = _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])), - xi_8); - const __m256d xi_10 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])); - const __m256d xi_11 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_12 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1])); - const __m256d xi_13 = _mm256_add_pd(_mm256_add_pd(xi_10, xi_11), xi_12); - const __m256d xi_14 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0])); - const __m256d xi_15 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - const __m256d xi_16 = _mm256_add_pd(xi_14, xi_15); - const __m256d xi_17 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_18 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0])); - const __m256d xi_19 = _mm256_add_pd(xi_17, xi_18); - const __m256d xi_20 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])); - const __m256d xi_21 = _mm256_add_pd(xi_10, xi_20); - const __m256d xi_22 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0])); - const __m256d xi_23 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0])); - const __m256d xi_24 = - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_17, xi_22), xi_23), - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - const __m256d xi_40 = - _mm256_mul_pd(_mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667), - _mm256_load_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_48 = - _mm256_mul_pd(_mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667), - _mm256_load_pd(&_data_force_20_30_10[ctr_0])); - const __m256d xi_52 = - _mm256_mul_pd(_mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667), - _mm256_load_pd(&_data_force_20_32_10[ctr_0])); - const __m256d xi_55 = - _mm256_mul_pd(_mm256_set_pd(0.5, 0.5, 0.5, 0.5), - _mm256_load_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_59 = - _mm256_mul_pd(_mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333), - _mm256_load_pd(&_data_force_20_30_10[ctr_0])); - const __m256d xi_63 = - _mm256_mul_pd(_mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333), - _mm256_load_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_73 = - _mm256_mul_pd(_mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333), - _mm256_load_pd(&_data_force_20_32_10[ctr_0])); - const __m256d xi_84 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0])); - const __m256d xi_85 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0])), - _mm256_mul_pd(_mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0]))), - xi_84); - const __m256d xi_86 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_load_pd( - &_data_pdfs_21_315_1m1[ctr_0])), - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_load_pd( - &_data_pdfs_21_316_11[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_load_pd( - &_data_pdfs_2m1_311_1m1[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0]))), - _mm256_mul_pd(_mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0]))), - xi_85), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_87 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_load_pd( - &_data_pdfs_21_315_1m1[ctr_0])), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_load_pd( - &_data_pdfs_21_316_11[ctr_0]))), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0]))), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0]))); - const __m256d xi_88 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1])), - _mm256_mul_pd( - _mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1]))), - xi_87); - const __m256d xi_89 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, - -5.0), - _mm256_loadu_pd( - &_data_pdfs_21_317_10[ctr_0 + - 1])), - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, - -5.0), - _mm256_loadu_pd( - &_data_pdfs_21_318_10[ctr_0 - - 1]))), - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, -5.0), - _mm256_loadu_pd( - &_data_pdfs_2m1_313_10[ctr_0 + - 1]))), - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, -5.0), - _mm256_loadu_pd( - &_data_pdfs_2m1_314_10[ctr_0 - 1]))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0), - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0), - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0]))), - xi_85), - xi_88), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_92 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - const __m256d xi_93 = _mm256_add_pd(xi_18, xi_92); - const __m256d xi_94 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])); - const __m256d xi_97 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - const __m256d xi_98 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_11, xi_15), xi_21), xi_97); - const __m256d xi_100 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_101 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - const __m256d xi_102 = _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1]))); - const __m256d xi_103 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd( - -7.0, -7.0, - -7.0, -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_310_11 - [ctr_0 - - 1])), - _mm256_mul_pd( - _mm256_set_pd( - -7.0, -7.0, - -7.0, -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_37_1m1 - [ctr_0 + - 1]))), - _mm256_mul_pd( - _mm256_set_pd( - -7.0, -7.0, - -7.0, -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_38_1m1 - [ctr_0 - - 1]))), - _mm256_mul_pd( - _mm256_set_pd( - -7.0, -7.0, -7.0, - -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_39_11 - [ctr_0 + 1]))), - _mm256_mul_pd( - _mm256_set_pd(-4.0, -4.0, - -4.0, -4.0), - _mm256_load_pd( - &_data_pdfs_21_36_10 - [ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-4.0, -4.0, -4.0, - -4.0), - _mm256_load_pd( - &_data_pdfs_2m1_35_10 - [ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_load_pd( - &_data_pdfs_20_31_1m1[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_load_pd( - &_data_pdfs_20_32_11[ctr_0]))), - xi_100), - xi_101), - xi_102), - xi_84), - xi_88), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_104 = - _mm256_add_pd(xi_92, _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0])); - const __m256d xi_105 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_104, xi_14), xi_22), - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0])), - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_107 = _mm256_mul_pd( - xi_105, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_108 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - const __m256d xi_109 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - const __m256d xi_110 = _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(-2.0, -2.0, -2.0, -2.0), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1]))); - const __m256d xi_111 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_108, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_109), - xi_110), - xi_14), - xi_19), - xi_2); - const __m256d xi_113 = _mm256_mul_pd( - xi_111, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_114 = - _mm256_mul_pd(xi_113, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_116 = _mm256_add_pd( - xi_94, _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - const __m256d xi_120 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1]), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_124 = _mm256_mul_pd( - xi_103, _mm256_set_pd(-0.0198412698412698, -0.0198412698412698, - -0.0198412698412698, -0.0198412698412698)); - const __m256d xi_132 = _mm256_add_pd( - xi_97, _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_133 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_12, xi_132), xi_20), - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1])), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])); - const __m256d xi_134 = _mm256_mul_pd( - xi_133, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_135 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_109, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_1), - xi_108), - xi_110), - xi_13); - const __m256d xi_136 = _mm256_mul_pd( - xi_135, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_138 = - _mm256_mul_pd(xi_136, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_139 = - _mm256_add_pd(_mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0]), - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_140 = - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_139, xi_23), xi_93), - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0])); - const __m256d xi_141 = _mm256_mul_pd( - xi_140, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_144 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_100, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_101, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_102), - xi_24), - xi_3); - const __m256d xi_145 = _mm256_mul_pd( - xi_144, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_146 = - _mm256_mul_pd(xi_145, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_148 = xi_145; - const __m256d xi_152 = _mm256_mul_pd( - xi_103, _mm256_set_pd(0.0138888888888889, 0.0138888888888889, - 0.0138888888888889, 0.0138888888888889)); - const __m256d xi_168 = _mm256_mul_pd( - xi_89, _mm256_set_pd(-0.00714285714285714, -0.00714285714285714, - -0.00714285714285714, -0.00714285714285714)); - const __m256d xi_170 = - _mm256_mul_pd(xi_86, _mm256_set_pd(0.025, 0.025, 0.025, 0.025)); - const __m256d xi_173 = _mm256_mul_pd( - xi_144, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d xi_175 = _mm256_mul_pd( - xi_140, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_176 = _mm256_mul_pd( - xi_103, _mm256_set_pd(-0.00396825396825397, -0.00396825396825397, - -0.00396825396825397, -0.00396825396825397)); - const __m256d xi_180 = _mm256_mul_pd( - xi_111, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d xi_181 = _mm256_mul_pd( - xi_105, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_187 = _mm256_mul_pd( - xi_89, _mm256_set_pd(0.0178571428571429, 0.0178571428571429, - 0.0178571428571429, 0.0178571428571429)); - const __m256d xi_190 = _mm256_mul_pd( - xi_133, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_191 = _mm256_mul_pd( - xi_135, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d vel0Term = _mm256_add_pd( - _mm256_add_pd(xi_1, - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])); - const __m256d vel1Term = _mm256_add_pd( - xi_2, _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - const __m256d vel2Term = _mm256_add_pd( - xi_3, _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d rho = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel0Term, vel1Term), - vel2Term), - xi_4), - xi_5), - xi_6), - _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0])); - const __m256d xi_7 = - _mm256_div_pd(_mm256_set_pd(1.0, 1.0, 1.0, 1.0), rho); - const __m256d u_0 = _mm256_mul_pd( - xi_7, _mm256_add_pd(_mm256_add_pd(vel0Term, xi_13), xi_9)); - const __m256d xi_25 = - _mm256_mul_pd(u_0, _mm256_load_pd(&_data_force_20_30_10[ctr_0])); - const __m256d xi_26 = _mm256_mul_pd( - xi_25, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_32 = - _mm256_mul_pd(xi_26, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_90 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_0, u_0))); - const __m256d xi_129 = _mm256_mul_pd(rho, u_0); - const __m256d xi_130 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(vel0Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_120), - xi_129), - xi_4), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - const __m256d xi_131 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_158 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d u_1 = _mm256_mul_pd( - xi_7, - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel1Term, xi_16), xi_19), xi_8), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1]))); - const __m256d xi_27 = - _mm256_mul_pd(u_1, _mm256_load_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_28 = _mm256_mul_pd( - xi_27, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_33 = - _mm256_mul_pd(xi_28, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_54 = - _mm256_mul_pd(u_1, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_57 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_mul_pd(u_0, xi_55), - _mm256_mul_pd(xi_54, - _mm256_load_pd(&_data_force_20_30_10[ctr_0]))), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_58 = - _mm256_mul_pd(xi_57, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_95 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_1, u_1))); - const __m256d xi_96 = _mm256_add_pd(_mm256_add_pd(xi_9, xi_94), xi_95); - const __m256d xi_115 = _mm256_mul_pd(rho, u_1); - const __m256d xi_117 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(vel1Term, _mm256_set_pd(-1.0, -1.0, - -1.0, -1.0)), - xi_115), - xi_116), - xi_5), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])), - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0])); - const __m256d xi_119 = _mm256_mul_pd( - xi_117, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_150 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_0, xi_115), xi_116), - xi_8), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_155 = _mm256_mul_pd( - xi_117, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d xi_156 = xi_155; - const __m256d xi_157 = _mm256_add_pd(xi_113, xi_156); - const __m256d xi_166 = - _mm256_mul_pd(xi_155, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_167 = _mm256_add_pd(xi_114, xi_166); - const __m256d xi_182 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_180, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_156), - xi_181); - const __m256d xi_183 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_181, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_166), - xi_180); - const __m256d u_2 = _mm256_mul_pd( - xi_7, - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel2Term, xi_21), xi_24), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1]))); - const __m256d xi_29 = - _mm256_mul_pd(u_2, _mm256_load_pd(&_data_force_20_32_10[ctr_0])); - const __m256d xi_30 = _mm256_mul_pd( - xi_29, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_31 = - _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(xi_26, xi_28), xi_30), - _mm256_set_pd(-omega_bulk + 2.0, -omega_bulk + 2.0, - -omega_bulk + 2.0, -omega_bulk + 2.0)); - const __m256d xi_34 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_29, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_32), - xi_33); - const __m256d xi_37 = - _mm256_mul_pd(xi_30, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_38 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_27, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_32), - xi_37); - const __m256d xi_39 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_25, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_33), - xi_37); - const __m256d xi_42 = - _mm256_mul_pd(xi_34, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_43 = - _mm256_mul_pd(xi_42, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_44 = - _mm256_mul_pd(xi_39, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_45 = - _mm256_mul_pd(xi_44, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_47 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_38, _mm256_set_pd(xi_46, xi_46, xi_46, xi_46)), - xi_43), - xi_45); - const __m256d xi_49 = - _mm256_mul_pd(xi_38, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_50 = - _mm256_mul_pd(xi_49, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_51 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_39, _mm256_set_pd(xi_46, xi_46, xi_46, xi_46)), - xi_43), - xi_50); - const __m256d xi_53 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_34, _mm256_set_pd(xi_46, xi_46, xi_46, xi_46)), - xi_45), - xi_50); - const __m256d xi_60 = _mm256_add_pd( - _mm256_mul_pd(xi_59, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_44); - const __m256d xi_62 = _mm256_mul_pd( - _mm256_mul_pd(xi_34, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_64 = - _mm256_mul_pd(xi_31, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_65 = _mm256_add_pd(xi_49, xi_64); - const __m256d xi_66 = _mm256_add_pd(xi_63, xi_65); - const __m256d xi_67 = _mm256_add_pd(xi_62, xi_66); - const __m256d xi_68 = _mm256_add_pd(xi_44, xi_59); - const __m256d xi_69 = _mm256_add_pd( - _mm256_mul_pd(xi_63, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_65); - const __m256d xi_70 = _mm256_add_pd(xi_62, xi_69); - const __m256d xi_71 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_mul_pd(u_2, xi_55), - _mm256_mul_pd(xi_54, - _mm256_load_pd(&_data_force_20_32_10[ctr_0]))), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_72 = _mm256_mul_pd( - _mm256_mul_pd(xi_39, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_74 = _mm256_add_pd(xi_42, xi_73); - const __m256d xi_75 = _mm256_add_pd(xi_72, xi_74); - const __m256d xi_76 = - _mm256_mul_pd(xi_71, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_77 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_mul_pd(u_0, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)), - _mm256_load_pd(&_data_force_20_32_10[ctr_0])), - _mm256_mul_pd( - _mm256_mul_pd(u_2, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)), - _mm256_load_pd(&_data_force_20_30_10[ctr_0]))), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_78 = - _mm256_mul_pd(xi_77, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_79 = _mm256_mul_pd( - _mm256_mul_pd(xi_38, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_80 = _mm256_add_pd(_mm256_add_pd(xi_64, xi_74), xi_79); - const __m256d xi_81 = _mm256_add_pd( - _mm256_mul_pd(xi_73, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_42); - const __m256d xi_82 = _mm256_add_pd(xi_72, xi_81); - const __m256d xi_83 = _mm256_add_pd(_mm256_add_pd(xi_64, xi_79), xi_81); - const __m256d xi_91 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_2, u_2))); - const __m256d xi_99 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_17, xi_22), - xi_90), - xi_91), - xi_93), - xi_96), - xi_98), - _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0])), - _mm256_set_pd(omega_bulk, omega_bulk, omega_bulk, omega_bulk)); - const __m256d xi_121 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_91, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0])), - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0])); - const __m256d xi_122 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_load_pd( - &_data_pdfs_20_31_1m1[ctr_0])), - xi_0), - xi_120), - xi_121), - xi_16), - xi_96), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, omega_shear)); - const __m256d xi_123 = - _mm256_mul_pd(xi_122, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_125 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_90, _mm256_set_pd( - 2.0, 2.0, - 2.0, 2.0)), - _mm256_mul_pd( - xi_95, - _mm256_set_pd( - -1.0, -1.0, -1.0, - -1.0))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, - -2.0, -2.0), - _mm256_loadu_pd( - &_data_pdfs_20_33_10 - [ctr_0 + 1]))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, -2.0, - -2.0), - _mm256_loadu_pd( - &_data_pdfs_20_34_10[ctr_0 - - 1]))), - xi_121), - xi_87), - xi_9), - xi_94), - xi_98), - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0])), - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0])), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, omega_shear)); - const __m256d xi_126 = _mm256_mul_pd( - xi_125, _mm256_set_pd(-0.0416666666666667, -0.0416666666666667, - -0.0416666666666667, -0.0416666666666667)); - const __m256d xi_127 = _mm256_add_pd( - _mm256_mul_pd(xi_86, _mm256_set_pd(-0.05, -0.05, -0.05, -0.05)), - xi_126); - const __m256d xi_128 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_89, _mm256_set_pd(0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143)), - xi_123), - xi_124), - xi_127); - const __m256d xi_137 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_125, _mm256_set_pd(0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333)), - _mm256_mul_pd(xi_89, _mm256_set_pd(-0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857))), - xi_124); - const __m256d xi_142 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(rho, u_2), - _mm256_mul_pd( - vel2Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_139), - xi_6), - xi_92), - xi_97), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])); - const __m256d xi_143 = _mm256_mul_pd( - xi_142, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_147 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_103, _mm256_set_pd(0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159)), - _mm256_mul_pd(xi_123, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - _mm256_mul_pd(xi_89, _mm256_set_pd(-0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714))), - xi_127); - const __m256d xi_151 = _mm256_mul_pd( - xi_122, _mm256_set_pd(0.0625, 0.0625, 0.0625, 0.0625)); - const __m256d xi_153 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_150, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_151), - xi_152); - const __m256d xi_159 = _mm256_mul_pd( - xi_99, _mm256_set_pd(0.0416666666666667, 0.0416666666666667, - 0.0416666666666667, 0.0416666666666667)); - const __m256d xi_160 = _mm256_add_pd( - _mm256_mul_pd( - xi_125, _mm256_set_pd(0.0208333333333333, 0.0208333333333333, - 0.0208333333333333, 0.0208333333333333)), - xi_159); - const __m256d xi_161 = _mm256_add_pd( - _mm256_mul_pd(xi_158, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_160); - const __m256d xi_162 = _mm256_add_pd(xi_138, xi_161); - const __m256d xi_163 = - _mm256_add_pd(_mm256_add_pd(xi_150, xi_151), xi_152); - const __m256d xi_164 = _mm256_add_pd(xi_158, xi_160); - const __m256d xi_165 = _mm256_add_pd(xi_136, xi_164); - const __m256d xi_169 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_115), xi_104), - xi_17), - _mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0])), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_171 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_126, xi_159), xi_168), - xi_169), - xi_170); - const __m256d xi_177 = _mm256_mul_pd( - xi_142, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d xi_178 = _mm256_add_pd(xi_176, xi_177); - const __m256d xi_179 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_173, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_175), - xi_178); - const __m256d xi_184 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_169, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_126), - xi_159), - xi_168), - xi_170); - const __m256d xi_185 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_129), xi_10), - xi_132), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_186 = - _mm256_mul_pd(xi_151, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_188 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_185, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_186), - xi_187); - const __m256d xi_189 = _mm256_add_pd(xi_148, xi_178); - const __m256d xi_192 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_190, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_161), - xi_191); - const __m256d xi_193 = - _mm256_add_pd(_mm256_add_pd(xi_185, xi_186), xi_187); - const __m256d xi_194 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_191, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_164), - xi_190); - const __m256d xi_195 = _mm256_add_pd( - _mm256_mul_pd(xi_177, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_176); - const __m256d xi_196 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_175, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_173), - xi_195); - const __m256d xi_197 = _mm256_add_pd(xi_146, xi_195); - const __m256d forceTerm_0 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_31, _mm256_set_pd(-1.5, -1.5, -1.5, -1.5)), - _mm256_mul_pd( - _mm256_mul_pd(xi_34, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))), - _mm256_mul_pd( - _mm256_mul_pd(xi_38, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))), - _mm256_mul_pd( - _mm256_mul_pd(xi_39, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))); - const __m256d forceTerm_1 = _mm256_add_pd(xi_40, xi_47); - const __m256d forceTerm_2 = _mm256_add_pd( - _mm256_mul_pd(xi_40, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_47); - const __m256d forceTerm_3 = _mm256_add_pd( - _mm256_mul_pd(xi_48, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_51); - const __m256d forceTerm_4 = _mm256_add_pd(xi_48, xi_51); - const __m256d forceTerm_5 = _mm256_add_pd(xi_52, xi_53); - const __m256d forceTerm_6 = _mm256_add_pd( - _mm256_mul_pd(xi_52, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_53); - const __m256d forceTerm_7 = - _mm256_add_pd(_mm256_add_pd(xi_58, xi_60), xi_67); - const __m256d forceTerm_8 = - _mm256_add_pd(_mm256_add_pd(xi_57, xi_67), xi_68); - const __m256d forceTerm_9 = - _mm256_add_pd(_mm256_add_pd(xi_57, xi_60), xi_70); - const __m256d forceTerm_10 = - _mm256_add_pd(_mm256_add_pd(xi_58, xi_68), xi_70); - const __m256d forceTerm_11 = - _mm256_add_pd(_mm256_add_pd(xi_66, xi_71), xi_75); - const __m256d forceTerm_12 = - _mm256_add_pd(_mm256_add_pd(xi_69, xi_75), xi_76); - const __m256d forceTerm_13 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_78), xi_80); - const __m256d forceTerm_14 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_77), xi_80); - const __m256d forceTerm_15 = - _mm256_add_pd(_mm256_add_pd(xi_66, xi_76), xi_82); - const __m256d forceTerm_16 = - _mm256_add_pd(_mm256_add_pd(xi_69, xi_71), xi_82); - const __m256d forceTerm_17 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_77), xi_83); - const __m256d forceTerm_18 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_78), xi_83); - _mm256_store_pd( - &_data_pdfs_tmp_20_30_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_103, _mm256_set_pd(0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238)), - _mm256_mul_pd( - xi_86, _mm256_set_pd(0.1, 0.1, 0.1, 0.1))), - _mm256_mul_pd(xi_89, - _mm256_set_pd(0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429))), - _mm256_mul_pd(xi_99, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - forceTerm_0), - _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_31_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_107, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_1), - xi_114), - xi_119), - xi_128), - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_32_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_119, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_2), - xi_107), - xi_113), - xi_128), - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_33_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_131, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_3), - xi_134), - xi_136), - xi_137), - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_34_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_134, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_4), - xi_131), - xi_137), - xi_138), - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_35_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_141, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_5), - xi_143), - xi_146), - xi_147), - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_36_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_143, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_6), - xi_141), - xi_147), - xi_148), - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_37_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_7, xi_153), xi_157), - xi_162), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_38_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_8, xi_157), xi_163), - xi_165), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_39_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_9, xi_162), xi_163), - xi_167), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_310_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_10, xi_153), xi_165), - xi_167), - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_311_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_11, xi_171), xi_179), - xi_182), - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_312_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_12, xi_179), xi_183), - xi_184), - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_313_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_13, xi_188), xi_189), - xi_192), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_314_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_14, xi_189), xi_193), - xi_194), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_315_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_15, xi_182), xi_184), - xi_196), - _mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_316_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_16, xi_171), xi_183), - xi_196), - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_317_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_17, xi_192), xi_193), - xi_197), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1]))); - _mm256_store_pd( - &_data_pdfs_tmp_20_318_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_18, xi_188), xi_194), - xi_197), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1]))); +static FUNC_PREFIX void kernel_streamCollide(double * RESTRICT const _data_force, double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, double omega_bulk, double omega_even, double omega_odd, double omega_shear) +{ + const double xi_50 = -omega_shear + 2.0; + const double xi_51 = xi_50*0.5; + const double xi_56 = xi_50*0.0833333333333333; + const double xi_61 = xi_50*0.166666666666667; + const double xi_71 = xi_50*0.25; + const double xi_76 = xi_50*0.0416666666666667; + const double xi_120 = omega_odd*0.25; + const double xi_126 = omega_odd*0.0833333333333333; + const double xi_159 = omega_shear*0.25; + const double xi_181 = omega_odd*0.0416666666666667; + const double xi_183 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_131 = rr_0*0.166666666666667; + const double xi_164 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + for (int64_t ctr_0 = 1; ctr_0 < ((_size_force_0 - 2) % (4) == 0 ? _size_force_0 - 2 : ((int64_t)((_size_force_0 - 2) / (4))+1) * (4)) + 1; ctr_0 += 4) + { + const __m256d xi_0 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1]),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + const __m256d xi_1 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_2 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_1),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])); + const __m256d xi_3 = _mm256_add_pd(_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0]),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_4 = _mm256_add_pd(xi_3,_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0])); + const __m256d xi_5 = _mm256_add_pd(xi_4,_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_6 = _mm256_add_pd(_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0]),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0])); + const __m256d xi_7 = _mm256_add_pd(xi_6,_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_8 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1]),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + const __m256d xi_9 = _mm256_add_pd(_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0]),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_10 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1]),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0])); + const __m256d xi_12 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])); + const __m256d xi_13 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1])); + const __m256d xi_14 = _mm256_add_pd(xi_12,xi_13); + const __m256d xi_15 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_16 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_17 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + const __m256d xi_18 = _mm256_add_pd(xi_16,xi_17); + const __m256d xi_19 = _mm256_add_pd(xi_15,xi_18); + const __m256d xi_20 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + const __m256d xi_21 = _mm256_add_pd(xi_20,_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + const __m256d xi_22 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0])); + const __m256d xi_23 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0])); + const __m256d xi_24 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_25 = _mm256_add_pd(_mm256_add_pd(xi_22,xi_23),xi_24); + const __m256d xi_26 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + const __m256d xi_27 = _mm256_add_pd(xi_12,xi_26); + const __m256d xi_28 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0])); + const __m256d xi_29 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0])); + const __m256d xi_30 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_24,xi_28),xi_29),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_31 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1]),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_32 = _mm256_add_pd(_mm256_add_pd(xi_31,xi_8),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_33 = _mm256_add_pd(_mm256_add_pd(xi_16,xi_21),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + const __m256d xi_34 = _mm256_add_pd(xi_26,_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])); + const __m256d xi_35 = _mm256_add_pd(xi_15,_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_36 = _mm256_add_pd(xi_34,xi_35); + const __m256d xi_37 = _mm256_add_pd(_mm256_add_pd(xi_9,_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0])); + const __m256d xi_38 = _mm256_add_pd(xi_28,_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_39 = _mm256_add_pd(_mm256_add_pd(xi_22,xi_38),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_55 = _mm256_mul_pd(_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667),_mm256_load_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_63 = _mm256_mul_pd(_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667),_mm256_load_pd(& _data_force_20_30_10[ctr_0])); + const __m256d xi_67 = _mm256_mul_pd(_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667),_mm256_load_pd(& _data_force_20_32_10[ctr_0])); + const __m256d xi_70 = _mm256_mul_pd(_mm256_set_pd(0.5,0.5,0.5,0.5),_mm256_load_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_74 = _mm256_mul_pd(_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333),_mm256_load_pd(& _data_force_20_30_10[ctr_0])); + const __m256d xi_78 = _mm256_mul_pd(_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333),_mm256_load_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_88 = _mm256_mul_pd(_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333),_mm256_load_pd(& _data_force_20_32_10[ctr_0])); + const __m256d xi_99 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_20_30_10[ctr_0])); + const __m256d xi_100 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0])),_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0]))),xi_99); + const __m256d xi_101 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0])),_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0]))),xi_100),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_102 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0])),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0]))); + const __m256d xi_103 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1])),_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1]))),xi_102); + const __m256d xi_104 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])),_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]))),_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1]))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0]))),xi_100),xi_103),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_107 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_108 = _mm256_add_pd(xi_107,xi_22); + const __m256d xi_109 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + const __m256d xi_112 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_113 = _mm256_add_pd(_mm256_add_pd(xi_112,xi_19),xi_27); + const __m256d xi_115 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_116 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_117 = _mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]))); + const __m256d xi_118 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])),_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1]))),_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-4.0,-4.0,-4.0,-4.0),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-4.0,-4.0,-4.0,-4.0),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0]))),xi_103),xi_115),xi_116),xi_117),xi_99),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_119 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_107,xi_23),xi_38),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0])),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0])); + const __m256d xi_121 = _mm256_mul_pd(xi_119,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_122 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_123 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + const __m256d xi_124 = _mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1]))); + const __m256d xi_125 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_122,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_123),xi_124),xi_25),xi_4); + const __m256d xi_127 = _mm256_mul_pd(xi_125,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_128 = _mm256_mul_pd(xi_127,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_133 = _mm256_mul_pd(xi_118,_mm256_set_pd(-0.0198412698412698,-0.0198412698412698,-0.0198412698412698,-0.0198412698412698)); + const __m256d xi_144 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_112,xi_13),xi_34),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_145 = _mm256_mul_pd(xi_144,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_146 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_123,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_122),xi_124),xi_14),xi_35),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + const __m256d xi_147 = _mm256_mul_pd(xi_146,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_149 = _mm256_mul_pd(xi_147,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_150 = _mm256_add_pd(_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0]),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_151 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_108,xi_150),xi_29),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0])); + const __m256d xi_152 = _mm256_mul_pd(xi_151,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_153 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_115,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_116,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_117),xi_30),xi_6); + const __m256d xi_154 = _mm256_mul_pd(xi_153,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_155 = _mm256_mul_pd(xi_154,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_162 = _mm256_mul_pd(xi_118,_mm256_set_pd(0.0138888888888889,0.0138888888888889,0.0138888888888889,0.0138888888888889)); + const __m256d xi_177 = _mm256_mul_pd(xi_104,_mm256_set_pd(-0.00714285714285714,-0.00714285714285714,-0.00714285714285714,-0.00714285714285714)); + const __m256d xi_179 = _mm256_mul_pd(xi_101,_mm256_set_pd(0.025,0.025,0.025,0.025)); + const __m256d xi_182 = _mm256_mul_pd(xi_153,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d xi_184 = _mm256_mul_pd(xi_151,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_185 = _mm256_mul_pd(xi_118,_mm256_set_pd(-0.00396825396825397,-0.00396825396825397,-0.00396825396825397,-0.00396825396825397)); + const __m256d xi_189 = _mm256_mul_pd(xi_125,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d xi_190 = _mm256_mul_pd(xi_119,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_196 = _mm256_mul_pd(xi_104,_mm256_set_pd(0.0178571428571429,0.0178571428571429,0.0178571428571429,0.0178571428571429)); + const __m256d xi_199 = _mm256_mul_pd(xi_144,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_200 = _mm256_mul_pd(xi_146,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d vel0Term = xi_2; + const __m256d vel1Term = xi_5; + const __m256d vel2Term = xi_7; + const __m256d rho = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel0Term,vel1Term),vel2Term),xi_10),xi_8),xi_9),_mm256_load_pd(& _data_pdfs_20_30_10[ctr_0])); + const __m256d xi_11 = _mm256_div_pd(_mm256_set_pd(1.0,1.0,1.0,1.0),rho); + const __m256d u_0 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(vel0Term,xi_14),xi_19)); + const __m256d xi_40 = _mm256_mul_pd(u_0,_mm256_load_pd(& _data_force_20_30_10[ctr_0])); + const __m256d xi_41 = _mm256_mul_pd(xi_40,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_47 = _mm256_mul_pd(xi_41,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_105 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_0,u_0))); + const __m256d xi_141 = _mm256_mul_pd(rho,u_0); + const __m256d xi_142 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel0Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_141),xi_32); + const __m256d xi_143 = _mm256_mul_pd(xi_142,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_167 = _mm256_mul_pd(xi_142,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d u_1 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel1Term,xi_17),xi_21),xi_25)); + const __m256d xi_42 = _mm256_mul_pd(u_1,_mm256_load_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_43 = _mm256_mul_pd(xi_42,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_48 = _mm256_mul_pd(xi_43,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_69 = _mm256_mul_pd(u_1,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_72 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_0,xi_70),_mm256_mul_pd(xi_69,_mm256_load_pd(& _data_force_20_30_10[ctr_0]))),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_73 = _mm256_mul_pd(xi_72,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_110 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_1,u_1))); + const __m256d xi_111 = _mm256_add_pd(_mm256_add_pd(xi_109,xi_110),xi_20); + const __m256d xi_129 = _mm256_mul_pd(rho,u_1); + const __m256d xi_130 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel1Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_109),xi_129),xi_37),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + const __m256d xi_132 = _mm256_mul_pd(xi_130,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_165 = _mm256_mul_pd(xi_130,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d xi_166 = _mm256_add_pd(xi_127,xi_165); + const __m256d xi_175 = _mm256_mul_pd(xi_165,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_176 = _mm256_add_pd(xi_128,xi_175); + const __m256d xi_191 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_189,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_165),xi_190); + const __m256d xi_192 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_190,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_175),xi_189); + const __m256d u_2 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel2Term,xi_27),xi_30),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1]))); + const __m256d xi_44 = _mm256_mul_pd(u_2,_mm256_load_pd(& _data_force_20_32_10[ctr_0])); + const __m256d xi_45 = _mm256_mul_pd(xi_44,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_46 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(xi_41,xi_43),xi_45),_mm256_set_pd(-omega_bulk + 2.0,-omega_bulk + 2.0,-omega_bulk + 2.0,-omega_bulk + 2.0)); + const __m256d xi_49 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_44,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_47),xi_48); + const __m256d xi_52 = _mm256_mul_pd(xi_45,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_53 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_42,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_47),xi_52); + const __m256d xi_54 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_40,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_48),xi_52); + const __m256d xi_57 = _mm256_mul_pd(xi_49,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_58 = _mm256_mul_pd(xi_57,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_59 = _mm256_mul_pd(xi_54,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_60 = _mm256_mul_pd(xi_59,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_62 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_58),xi_60); + const __m256d xi_64 = _mm256_mul_pd(xi_53,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_65 = _mm256_mul_pd(xi_64,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_66 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_58),xi_65); + const __m256d xi_68 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_60),xi_65); + const __m256d xi_75 = _mm256_add_pd(_mm256_mul_pd(xi_74,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_59); + const __m256d xi_77 = _mm256_mul_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_79 = _mm256_mul_pd(xi_46,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_80 = _mm256_add_pd(xi_64,xi_79); + const __m256d xi_81 = _mm256_add_pd(xi_78,xi_80); + const __m256d xi_82 = _mm256_add_pd(xi_77,xi_81); + const __m256d xi_83 = _mm256_add_pd(xi_59,xi_74); + const __m256d xi_84 = _mm256_add_pd(_mm256_mul_pd(xi_78,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_80); + const __m256d xi_85 = _mm256_add_pd(xi_77,xi_84); + const __m256d xi_86 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_2,xi_70),_mm256_mul_pd(xi_69,_mm256_load_pd(& _data_force_20_32_10[ctr_0]))),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_87 = _mm256_mul_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_89 = _mm256_add_pd(xi_57,xi_88); + const __m256d xi_90 = _mm256_add_pd(xi_87,xi_89); + const __m256d xi_91 = _mm256_mul_pd(xi_86,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_92 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_mul_pd(u_0,_mm256_set_pd(0.5,0.5,0.5,0.5)),_mm256_load_pd(& _data_force_20_32_10[ctr_0])),_mm256_mul_pd(_mm256_mul_pd(u_2,_mm256_set_pd(0.5,0.5,0.5,0.5)),_mm256_load_pd(& _data_force_20_30_10[ctr_0]))),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_93 = _mm256_mul_pd(xi_92,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_94 = _mm256_mul_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_95 = _mm256_add_pd(_mm256_add_pd(xi_79,xi_89),xi_94); + const __m256d xi_96 = _mm256_add_pd(_mm256_mul_pd(xi_88,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_57); + const __m256d xi_97 = _mm256_add_pd(xi_87,xi_96); + const __m256d xi_98 = _mm256_add_pd(_mm256_add_pd(xi_79,xi_94),xi_96); + const __m256d xi_106 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_2,u_2))); + const __m256d xi_114 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_105,xi_106),xi_108),xi_111),xi_113),xi_24),xi_28),_mm256_load_pd(& _data_pdfs_20_30_10[ctr_0])),_mm256_set_pd(omega_bulk,omega_bulk,omega_bulk,omega_bulk)); + const __m256d xi_134 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_106,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0])),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0])); + const __m256d xi_135 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0])),xi_1),xi_111),xi_134),xi_18),xi_23),xi_31),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_136 = _mm256_mul_pd(xi_135,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_137 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_105,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_110,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1]))),xi_102),xi_109),xi_113),xi_134),xi_20),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0])),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0])),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_138 = _mm256_mul_pd(xi_137,_mm256_set_pd(-0.0416666666666667,-0.0416666666666667,-0.0416666666666667,-0.0416666666666667)); + const __m256d xi_139 = _mm256_add_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-0.05,-0.05,-0.05,-0.05)),xi_138); + const __m256d xi_140 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(0.0142857142857143,0.0142857142857143,0.0142857142857143,0.0142857142857143)),xi_133),xi_136),xi_139); + const __m256d xi_148 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(-0.0357142857142857,-0.0357142857142857,-0.0357142857142857,-0.0357142857142857)),_mm256_mul_pd(xi_137,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333))),xi_133); + const __m256d xi_156 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(rho,u_2),_mm256_mul_pd(vel2Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_10),xi_107),xi_112),xi_150),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + const __m256d xi_157 = _mm256_mul_pd(xi_156,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_158 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(-0.0214285714285714,-0.0214285714285714,-0.0214285714285714,-0.0214285714285714)),_mm256_mul_pd(xi_118,_mm256_set_pd(0.0158730158730159,0.0158730158730159,0.0158730158730159,0.0158730158730159))),_mm256_mul_pd(xi_136,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_139); + const __m256d xi_161 = _mm256_mul_pd(xi_135,_mm256_set_pd(0.0625,0.0625,0.0625,0.0625)); + const __m256d xi_168 = _mm256_mul_pd(xi_114,_mm256_set_pd(0.0416666666666667,0.0416666666666667,0.0416666666666667,0.0416666666666667)); + const __m256d xi_169 = _mm256_add_pd(_mm256_mul_pd(xi_137,_mm256_set_pd(0.0208333333333333,0.0208333333333333,0.0208333333333333,0.0208333333333333)),xi_168); + const __m256d xi_170 = _mm256_add_pd(_mm256_mul_pd(xi_167,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_169); + const __m256d xi_171 = _mm256_add_pd(xi_149,xi_170); + const __m256d xi_173 = _mm256_add_pd(xi_167,xi_169); + const __m256d xi_174 = _mm256_add_pd(xi_147,xi_173); + const __m256d xi_186 = _mm256_mul_pd(xi_156,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d xi_187 = _mm256_add_pd(xi_185,xi_186); + const __m256d xi_188 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_182,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_184),xi_187); + const __m256d xi_195 = _mm256_mul_pd(xi_161,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_198 = _mm256_add_pd(xi_154,xi_187); + const __m256d xi_201 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_199,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_170),xi_200); + const __m256d xi_203 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_200,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_173),xi_199); + const __m256d xi_204 = _mm256_add_pd(_mm256_mul_pd(xi_186,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_185); + const __m256d xi_205 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_184,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_182),xi_204); + const __m256d xi_206 = _mm256_add_pd(xi_155,xi_204); + const __m256d p_0 = _mm256_add_pd(xi_2,xi_32); + const __m256d p_1 = xi_33; + const __m256d xi_160 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_1,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_0,xi_129)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_163 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_160,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_161),xi_162); + const __m256d xi_172 = _mm256_add_pd(_mm256_add_pd(xi_160,xi_161),xi_162); + const __m256d p_2 = xi_36; + const __m256d xi_194 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_2,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_141)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_197 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_194,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_195),xi_196); + const __m256d xi_202 = _mm256_add_pd(_mm256_add_pd(xi_194,xi_195),xi_196); + const __m256d p_3 = xi_33; + const __m256d p_4 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_37),xi_5); + const __m256d p_5 = xi_39; + const __m256d xi_178 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_5,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_129)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_180 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_138,xi_168),xi_177),xi_178),xi_179); + const __m256d xi_193 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_178,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_138),xi_168),xi_177),xi_179); + const __m256d p_6 = xi_36; + const __m256d p_7 = xi_39; + const __m256d p_8 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_1,xi_10),xi_3),xi_7),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d forceTerm_0 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_46,_mm256_set_pd(-1.5,-1.5,-1.5,-1.5)),_mm256_mul_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))),_mm256_mul_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))),_mm256_mul_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))); + const __m256d forceTerm_1 = _mm256_add_pd(xi_55,xi_62); + const __m256d forceTerm_2 = _mm256_add_pd(_mm256_mul_pd(xi_55,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_62); + const __m256d forceTerm_3 = _mm256_add_pd(_mm256_mul_pd(xi_63,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_66); + const __m256d forceTerm_4 = _mm256_add_pd(xi_63,xi_66); + const __m256d forceTerm_5 = _mm256_add_pd(xi_67,xi_68); + const __m256d forceTerm_6 = _mm256_add_pd(_mm256_mul_pd(xi_67,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_68); + const __m256d forceTerm_7 = _mm256_add_pd(_mm256_add_pd(xi_73,xi_75),xi_82); + const __m256d forceTerm_8 = _mm256_add_pd(_mm256_add_pd(xi_72,xi_82),xi_83); + const __m256d forceTerm_9 = _mm256_add_pd(_mm256_add_pd(xi_72,xi_75),xi_85); + const __m256d forceTerm_10 = _mm256_add_pd(_mm256_add_pd(xi_73,xi_83),xi_85); + const __m256d forceTerm_11 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_86),xi_90); + const __m256d forceTerm_12 = _mm256_add_pd(_mm256_add_pd(xi_84,xi_90),xi_91); + const __m256d forceTerm_13 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_93),xi_95); + const __m256d forceTerm_14 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_92),xi_95); + const __m256d forceTerm_15 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_91),xi_97); + const __m256d forceTerm_16 = _mm256_add_pd(_mm256_add_pd(xi_84,xi_86),xi_97); + const __m256d forceTerm_17 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_92),xi_98); + const __m256d forceTerm_18 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_93),xi_98); + _mm256_store_pd(&_data_pdfs_tmp_20_30_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(0.1,0.1,0.1,0.1)),_mm256_mul_pd(xi_104,_mm256_set_pd(0.0428571428571429,0.0428571428571429,0.0428571428571429,0.0428571428571429))),_mm256_mul_pd(xi_114,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_mul_pd(xi_118,_mm256_set_pd(0.0238095238095238,0.0238095238095238,0.0238095238095238,0.0238095238095238))),forceTerm_0),_mm256_load_pd(& _data_pdfs_20_30_10[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_31_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_121,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_1),xi_128),xi_132),xi_140),_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_32_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_132,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_2),xi_121),xi_127),xi_140),_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_33_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_143,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_3),xi_145),xi_147),xi_148),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_34_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_145,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_4),xi_143),xi_148),xi_149),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_35_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_152,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_5),xi_155),xi_157),xi_158),_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_36_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_157,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_6),xi_152),xi_154),xi_158),_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_37_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_7,xi_163),xi_166),xi_171),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_38_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_8,xi_166),xi_172),xi_174),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_39_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_9,xi_171),xi_172),xi_176),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_310_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_10,xi_163),xi_174),xi_176),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_311_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_11,xi_180),xi_188),xi_191),_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_312_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_12,xi_188),xi_192),xi_193),_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_313_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_13,xi_197),xi_198),xi_201),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_314_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_14,xi_198),xi_202),xi_203),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_315_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_15,xi_191),xi_193),xi_205),_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_316_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_16,xi_180),xi_192),xi_205),_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0]))); + _mm256_store_pd(&_data_pdfs_tmp_20_317_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_17,xi_201),xi_202),xi_206),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1]))); + _mm256_store_pd(&_data_pdfs_tmp_20_318_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_18,xi_197),xi_203),xi_206),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]))); + } } - } - } + } +} } -} // namespace internal_kernel_streamCollide namespace internal_kernel_collide { -static FUNC_PREFIX void -kernel_collide(double *RESTRICT const _data_force, double *RESTRICT _data_pdfs, - int64_t const _size_force_0, int64_t const _size_force_1, - int64_t const _size_force_2, int64_t const _stride_force_1, - int64_t const _stride_force_2, int64_t const _stride_force_3, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, double omega_bulk, - double omega_even, double omega_odd, double omega_shear) { - const double xi_35 = -omega_shear + 2.0; - const double xi_36 = xi_35 * 0.5; - const double xi_41 = xi_35 * 0.0833333333333333; - const double xi_46 = xi_35 * 0.166666666666667; - const double xi_56 = xi_35 * 0.25; - const double xi_61 = xi_35 * 0.0416666666666667; - const double xi_106 = omega_odd * 0.25; - const double xi_112 = omega_odd * 0.0833333333333333; - const double xi_149 = omega_shear * 0.25; - const double xi_172 = omega_odd * 0.0416666666666667; - const double xi_174 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_118 = rr_0 * 0.166666666666667; - const double xi_154 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) { - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_313 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_316 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_311 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_312 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_314 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_315 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_35 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_36 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_317 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_318 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) { - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_313; - double *RESTRICT _data_pdfs_20_316_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_316; - double *RESTRICT _data_pdfs_20_311_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_311; - double *RESTRICT _data_pdfs_20_310_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_20_38_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_20_312_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_312; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_20_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_314; - double *RESTRICT _data_pdfs_20_315_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_315; - double *RESTRICT _data_pdfs_20_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_35; - double *RESTRICT _data_pdfs_20_37_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_32_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_32; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_31_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_20_39_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_36; - double *RESTRICT _data_pdfs_20_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_317; - double *RESTRICT _data_pdfs_20_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_318; - for (int64_t ctr_0 = 0; - ctr_0 < ((_size_force_0) % (4) == 0 - ? _size_force_0 - : ((int64_t)((_size_force_0) / (4)) + 1) * (4)); - ctr_0 += 4) { - const __m256d xi_198 = _mm256_load_pd(&_data_force_20_31_10[ctr_0]); - const __m256d xi_199 = _mm256_load_pd(&_data_force_20_30_10[ctr_0]); - const __m256d xi_200 = _mm256_load_pd(&_data_pdfs_20_33_10[ctr_0]); - const __m256d xi_201 = _mm256_load_pd(&_data_pdfs_20_313_10[ctr_0]); - const __m256d xi_202 = _mm256_load_pd(&_data_pdfs_20_316_10[ctr_0]); - const __m256d xi_203 = _mm256_load_pd(&_data_pdfs_20_311_10[ctr_0]); - const __m256d xi_204 = _mm256_load_pd(&_data_pdfs_20_310_10[ctr_0]); - const __m256d xi_205 = _mm256_load_pd(&_data_pdfs_20_38_10[ctr_0]); - const __m256d xi_206 = _mm256_load_pd(&_data_pdfs_20_312_10[ctr_0]); - const __m256d xi_207 = _mm256_load_pd(&_data_pdfs_20_34_10[ctr_0]); - const __m256d xi_208 = _mm256_load_pd(&_data_pdfs_20_314_10[ctr_0]); - const __m256d xi_209 = _mm256_load_pd(&_data_pdfs_20_315_10[ctr_0]); - const __m256d xi_210 = _mm256_load_pd(&_data_pdfs_20_35_10[ctr_0]); - const __m256d xi_211 = _mm256_load_pd(&_data_pdfs_20_37_10[ctr_0]); - const __m256d xi_212 = _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0]); - const __m256d xi_213 = _mm256_load_pd(&_data_pdfs_20_32_10[ctr_0]); - const __m256d xi_214 = _mm256_load_pd(&_data_force_20_32_10[ctr_0]); - const __m256d xi_215 = _mm256_load_pd(&_data_pdfs_20_31_10[ctr_0]); - const __m256d xi_216 = _mm256_load_pd(&_data_pdfs_20_39_10[ctr_0]); - const __m256d xi_217 = _mm256_load_pd(&_data_pdfs_20_36_10[ctr_0]); - const __m256d xi_218 = _mm256_load_pd(&_data_pdfs_20_317_10[ctr_0]); - const __m256d xi_219 = _mm256_load_pd(&_data_pdfs_20_318_10[ctr_0]); - const __m256d xi_0 = _mm256_add_pd(xi_208, xi_219); - const __m256d xi_1 = _mm256_add_pd(xi_0, xi_207); - const __m256d xi_2 = - _mm256_add_pd(_mm256_add_pd(xi_203, xi_209), xi_215); - const __m256d xi_3 = _mm256_add_pd(xi_206, xi_210); - const __m256d xi_4 = _mm256_add_pd(xi_200, xi_216); - const __m256d xi_5 = _mm256_add_pd(xi_202, xi_213); - const __m256d xi_6 = _mm256_add_pd(xi_217, xi_218); - const __m256d xi_8 = - _mm256_mul_pd(xi_216, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_9 = _mm256_add_pd( - _mm256_mul_pd(xi_211, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_8); - const __m256d xi_10 = - _mm256_mul_pd(xi_218, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_11 = - _mm256_mul_pd(xi_201, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_12 = - _mm256_mul_pd(xi_200, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_13 = _mm256_add_pd(_mm256_add_pd(xi_10, xi_11), xi_12); - const __m256d xi_14 = - _mm256_mul_pd(xi_213, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_15 = - _mm256_mul_pd(xi_204, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_16 = _mm256_add_pd(xi_14, xi_15); - const __m256d xi_17 = - _mm256_mul_pd(xi_202, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_18 = - _mm256_mul_pd(xi_206, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_19 = _mm256_add_pd(xi_17, xi_18); - const __m256d xi_20 = - _mm256_mul_pd(xi_219, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_21 = _mm256_add_pd(xi_10, xi_20); - const __m256d xi_22 = - _mm256_mul_pd(xi_209, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_23 = - _mm256_mul_pd(xi_217, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_24 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_17, xi_203), xi_22), xi_23); - const __m256d xi_40 = _mm256_mul_pd( - xi_198, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_48 = _mm256_mul_pd( - xi_199, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_52 = _mm256_mul_pd( - xi_214, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_55 = - _mm256_mul_pd(xi_198, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_59 = _mm256_mul_pd( - xi_199, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_63 = _mm256_mul_pd( - xi_198, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_73 = _mm256_mul_pd( - xi_214, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_84 = - _mm256_mul_pd(xi_212, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_85 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_210, _mm256_set_pd(3.0, 3.0, 3.0, 3.0)), - _mm256_mul_pd(xi_217, _mm256_set_pd(3.0, 3.0, 3.0, 3.0))), - xi_84); - const __m256d xi_86 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_202, - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0)), - _mm256_mul_pd( - xi_203, - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0))), - _mm256_mul_pd( - xi_206, - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0))), - _mm256_mul_pd( - xi_209, _mm256_set_pd(-3.0, -3.0, -3.0, -3.0))), - _mm256_mul_pd(xi_213, - _mm256_set_pd(3.0, 3.0, 3.0, 3.0))), - _mm256_mul_pd(xi_215, _mm256_set_pd(3.0, 3.0, 3.0, 3.0))), - xi_85), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_87 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_202, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)), - _mm256_mul_pd(xi_203, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))), - _mm256_mul_pd(xi_206, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))), - _mm256_mul_pd(xi_209, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))); - const __m256d xi_88 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_200, _mm256_set_pd(5.0, 5.0, 5.0, 5.0)), - _mm256_mul_pd(xi_207, _mm256_set_pd(5.0, 5.0, 5.0, 5.0))), - xi_87); - const __m256d xi_89 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_201, _mm256_set_pd(-5.0, -5.0, - -5.0, -5.0)), - _mm256_mul_pd( - xi_208, _mm256_set_pd(-5.0, -5.0, - -5.0, -5.0))), - _mm256_mul_pd( - xi_213, - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0))), - _mm256_mul_pd( - xi_215, - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0))), - _mm256_mul_pd( - xi_218, _mm256_set_pd(-5.0, -5.0, -5.0, -5.0))), - _mm256_mul_pd(xi_219, - _mm256_set_pd(-5.0, -5.0, -5.0, -5.0))), - xi_85), - xi_88), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_92 = - _mm256_mul_pd(xi_203, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_93 = _mm256_add_pd(xi_18, xi_92); - const __m256d xi_94 = - _mm256_mul_pd(xi_205, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_97 = - _mm256_mul_pd(xi_208, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_98 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_11, xi_15), xi_21), xi_97); - const __m256d xi_100 = - _mm256_mul_pd(xi_201, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_101 = - _mm256_mul_pd(xi_208, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_102 = _mm256_add_pd( - _mm256_mul_pd(xi_218, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)), - _mm256_mul_pd(xi_219, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))); - const __m256d xi_103 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_204, - _mm256_set_pd( - -7.0, -7.0, - -7.0, - -7.0)), - _mm256_mul_pd( - xi_205, - _mm256_set_pd( - -7.0, -7.0, - -7.0, - -7.0))), - _mm256_mul_pd( - xi_210, - _mm256_set_pd( - -4.0, -4.0, - -4.0, -4.0))), - _mm256_mul_pd( - xi_211, - _mm256_set_pd( - -7.0, -7.0, -7.0, - -7.0))), - _mm256_mul_pd( - xi_213, - _mm256_set_pd(5.0, 5.0, 5.0, - 5.0))), - _mm256_mul_pd( - xi_215, - _mm256_set_pd(5.0, 5.0, 5.0, - 5.0))), - _mm256_mul_pd( - xi_216, _mm256_set_pd(-7.0, -7.0, - -7.0, -7.0))), - _mm256_mul_pd( - xi_217, - _mm256_set_pd(-4.0, -4.0, -4.0, -4.0))), - xi_100), - xi_101), - xi_102), - xi_84), - xi_88), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_104 = _mm256_add_pd(xi_206, xi_92); - const __m256d xi_105 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_104, xi_14), xi_202), - xi_215), - xi_22); - const __m256d xi_107 = _mm256_mul_pd( - xi_105, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_108 = - _mm256_mul_pd(xi_211, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_109 = - _mm256_mul_pd(xi_204, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_110 = _mm256_add_pd( - _mm256_mul_pd(xi_205, _mm256_set_pd(-2.0, -2.0, -2.0, -2.0)), - _mm256_mul_pd(xi_216, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))); - const __m256d xi_111 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_108, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_109), - xi_110), - xi_14), - xi_19), - xi_2); - const __m256d xi_113 = _mm256_mul_pd( - xi_111, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_114 = - _mm256_mul_pd(xi_113, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_116 = _mm256_add_pd(xi_204, xi_94); - const __m256d xi_120 = _mm256_add_pd(xi_201, xi_218); - const __m256d xi_124 = _mm256_mul_pd( - xi_103, _mm256_set_pd(-0.0198412698412698, -0.0198412698412698, - -0.0198412698412698, -0.0198412698412698)); - const __m256d xi_132 = _mm256_add_pd(xi_201, xi_97); - const __m256d xi_133 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_12, xi_132), xi_20), - xi_207), - xi_218); - const __m256d xi_134 = _mm256_mul_pd( - xi_133, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_135 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_109, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_1), - xi_108), - xi_110), - xi_13); - const __m256d xi_136 = _mm256_mul_pd( - xi_135, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_138 = - _mm256_mul_pd(xi_136, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_139 = _mm256_add_pd(xi_202, xi_209); - const __m256d xi_140 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_139, xi_210), xi_23), xi_93); - const __m256d xi_141 = _mm256_mul_pd( - xi_140, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_144 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_100, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_101, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_102), - xi_24), - xi_3); - const __m256d xi_145 = _mm256_mul_pd( - xi_144, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_146 = - _mm256_mul_pd(xi_145, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_148 = xi_145; - const __m256d xi_152 = _mm256_mul_pd( - xi_103, _mm256_set_pd(0.0138888888888889, 0.0138888888888889, - 0.0138888888888889, 0.0138888888888889)); - const __m256d xi_168 = _mm256_mul_pd( - xi_89, _mm256_set_pd(-0.00714285714285714, -0.00714285714285714, - -0.00714285714285714, -0.00714285714285714)); - const __m256d xi_170 = - _mm256_mul_pd(xi_86, _mm256_set_pd(0.025, 0.025, 0.025, 0.025)); - const __m256d xi_173 = _mm256_mul_pd( - xi_144, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d xi_175 = _mm256_mul_pd( - xi_140, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_176 = _mm256_mul_pd( - xi_103, _mm256_set_pd(-0.00396825396825397, -0.00396825396825397, - -0.00396825396825397, -0.00396825396825397)); - const __m256d xi_180 = _mm256_mul_pd( - xi_111, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d xi_181 = _mm256_mul_pd( - xi_105, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_187 = _mm256_mul_pd( - xi_89, _mm256_set_pd(0.0178571428571429, 0.0178571428571429, - 0.0178571428571429, 0.0178571428571429)); - const __m256d xi_190 = _mm256_mul_pd( - xi_133, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_191 = _mm256_mul_pd( - xi_135, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d vel0Term = - _mm256_add_pd(_mm256_add_pd(xi_1, xi_204), xi_205); - const __m256d vel1Term = _mm256_add_pd(xi_2, xi_211); - const __m256d vel2Term = _mm256_add_pd(xi_201, xi_3); - const __m256d rho = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel0Term, vel1Term), - vel2Term), - xi_212), - xi_4), - xi_5), - xi_6); - const __m256d xi_7 = - _mm256_div_pd(_mm256_set_pd(1.0, 1.0, 1.0, 1.0), rho); - const __m256d u_0 = _mm256_mul_pd( - xi_7, _mm256_add_pd(_mm256_add_pd(vel0Term, xi_13), xi_9)); - const __m256d xi_25 = _mm256_mul_pd(u_0, xi_199); - const __m256d xi_26 = _mm256_mul_pd( - xi_25, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_32 = - _mm256_mul_pd(xi_26, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_90 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_0, u_0))); - const __m256d xi_129 = _mm256_mul_pd(rho, u_0); - const __m256d xi_130 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(vel0Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_120), - xi_129), - xi_211), - xi_4); - const __m256d xi_131 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_158 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d u_1 = _mm256_mul_pd( - xi_7, _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel1Term, xi_16), xi_19), - xi_205), - xi_8)); - const __m256d xi_27 = _mm256_mul_pd(u_1, xi_198); - const __m256d xi_28 = _mm256_mul_pd( - xi_27, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_33 = - _mm256_mul_pd(xi_28, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_54 = - _mm256_mul_pd(u_1, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_57 = - _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_0, xi_55), - _mm256_mul_pd(xi_199, xi_54)), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_58 = - _mm256_mul_pd(xi_57, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_95 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_1, u_1))); - const __m256d xi_96 = _mm256_add_pd(_mm256_add_pd(xi_9, xi_94), xi_95); - const __m256d xi_115 = _mm256_mul_pd(rho, u_1); - const __m256d xi_117 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(vel1Term, _mm256_set_pd(-1.0, -1.0, - -1.0, -1.0)), - xi_115), - xi_116), - xi_206), - xi_216), - xi_5); - const __m256d xi_119 = _mm256_mul_pd( - xi_117, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_150 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_0, xi_115), xi_116), - xi_211), - xi_8), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_155 = _mm256_mul_pd( - xi_117, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d xi_156 = xi_155; - const __m256d xi_157 = _mm256_add_pd(xi_113, xi_156); - const __m256d xi_166 = - _mm256_mul_pd(xi_155, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_167 = _mm256_add_pd(xi_114, xi_166); - const __m256d xi_182 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_180, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_156), - xi_181); - const __m256d xi_183 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_181, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_166), - xi_180); - const __m256d u_2 = _mm256_mul_pd( - xi_7, - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel2Term, xi_208), xi_21), - xi_24)); - const __m256d xi_29 = _mm256_mul_pd(u_2, xi_214); - const __m256d xi_30 = _mm256_mul_pd( - xi_29, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_31 = - _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(xi_26, xi_28), xi_30), - _mm256_set_pd(-omega_bulk + 2.0, -omega_bulk + 2.0, - -omega_bulk + 2.0, -omega_bulk + 2.0)); - const __m256d xi_34 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_29, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_32), - xi_33); - const __m256d xi_37 = - _mm256_mul_pd(xi_30, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_38 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_27, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_32), - xi_37); - const __m256d xi_39 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_25, _mm256_set_pd(0.666666666666667, 0.666666666666667, - 0.666666666666667, 0.666666666666667)), - xi_33), - xi_37); - const __m256d xi_42 = - _mm256_mul_pd(xi_34, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_43 = - _mm256_mul_pd(xi_42, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_44 = - _mm256_mul_pd(xi_39, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_45 = - _mm256_mul_pd(xi_44, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_47 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_38, _mm256_set_pd(xi_46, xi_46, xi_46, xi_46)), - xi_43), - xi_45); - const __m256d xi_49 = - _mm256_mul_pd(xi_38, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_50 = - _mm256_mul_pd(xi_49, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_51 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_39, _mm256_set_pd(xi_46, xi_46, xi_46, xi_46)), - xi_43), - xi_50); - const __m256d xi_53 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_34, _mm256_set_pd(xi_46, xi_46, xi_46, xi_46)), - xi_45), - xi_50); - const __m256d xi_60 = _mm256_add_pd( - _mm256_mul_pd(xi_59, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_44); - const __m256d xi_62 = _mm256_mul_pd( - _mm256_mul_pd(xi_34, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_64 = - _mm256_mul_pd(xi_31, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_65 = _mm256_add_pd(xi_49, xi_64); - const __m256d xi_66 = _mm256_add_pd(xi_63, xi_65); - const __m256d xi_67 = _mm256_add_pd(xi_62, xi_66); - const __m256d xi_68 = _mm256_add_pd(xi_44, xi_59); - const __m256d xi_69 = _mm256_add_pd( - _mm256_mul_pd(xi_63, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_65); - const __m256d xi_70 = _mm256_add_pd(xi_62, xi_69); - const __m256d xi_71 = - _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_55), - _mm256_mul_pd(xi_214, xi_54)), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_72 = _mm256_mul_pd( - _mm256_mul_pd(xi_39, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_74 = _mm256_add_pd(xi_42, xi_73); - const __m256d xi_75 = _mm256_add_pd(xi_72, xi_74); - const __m256d xi_76 = - _mm256_mul_pd(xi_71, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_77 = _mm256_mul_pd( - _mm256_add_pd(_mm256_mul_pd(_mm256_mul_pd(u_0, xi_214), - _mm256_set_pd(0.5, 0.5, 0.5, 0.5)), - _mm256_mul_pd(_mm256_mul_pd(u_2, xi_199), - _mm256_set_pd(0.5, 0.5, 0.5, 0.5))), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_78 = - _mm256_mul_pd(xi_77, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_79 = _mm256_mul_pd( - _mm256_mul_pd(xi_38, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_80 = _mm256_add_pd(_mm256_add_pd(xi_64, xi_74), xi_79); - const __m256d xi_81 = _mm256_add_pd( - _mm256_mul_pd(xi_73, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_42); - const __m256d xi_82 = _mm256_add_pd(xi_72, xi_81); - const __m256d xi_83 = _mm256_add_pd(_mm256_add_pd(xi_64, xi_79), xi_81); - const __m256d xi_91 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_2, u_2))); - const __m256d xi_99 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_17, xi_212), - xi_22), - xi_90), - xi_91), - xi_93), - xi_96), - xi_98), - _mm256_set_pd(omega_bulk, omega_bulk, omega_bulk, omega_bulk)); - const __m256d xi_121 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_91, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_210), - xi_217); - const __m256d xi_122 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_215, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_0), - xi_120), - xi_121), - xi_16), - xi_96), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, omega_shear)); - const __m256d xi_123 = - _mm256_mul_pd(xi_122, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_125 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_200, - _mm256_set_pd( - -2.0, -2.0, -2.0, - -2.0)), - _mm256_mul_pd( - xi_207, - _mm256_set_pd( - -2.0, -2.0, -2.0, - -2.0))), - _mm256_mul_pd( - xi_90, - _mm256_set_pd(2.0, 2.0, 2.0, - 2.0))), - _mm256_mul_pd( - xi_95, - _mm256_set_pd(-1.0, -1.0, -1.0, - -1.0))), - xi_121), - xi_213), - xi_215), - xi_87), - xi_9), - xi_94), - xi_98), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, omega_shear)); - const __m256d xi_126 = _mm256_mul_pd( - xi_125, _mm256_set_pd(-0.0416666666666667, -0.0416666666666667, - -0.0416666666666667, -0.0416666666666667)); - const __m256d xi_127 = _mm256_add_pd( - _mm256_mul_pd(xi_86, _mm256_set_pd(-0.05, -0.05, -0.05, -0.05)), - xi_126); - const __m256d xi_128 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_89, _mm256_set_pd(0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143)), - xi_123), - xi_124), - xi_127); - const __m256d xi_137 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_125, _mm256_set_pd(0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333)), - _mm256_mul_pd(xi_89, _mm256_set_pd(-0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857))), - xi_124); - const __m256d xi_142 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(rho, u_2), - _mm256_mul_pd( - vel2Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_139), - xi_219), - xi_6), - xi_92), - xi_97); - const __m256d xi_143 = _mm256_mul_pd( - xi_142, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_147 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_103, _mm256_set_pd(0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159)), - _mm256_mul_pd(xi_123, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - _mm256_mul_pd(xi_89, _mm256_set_pd(-0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714))), - xi_127); - const __m256d xi_151 = _mm256_mul_pd( - xi_122, _mm256_set_pd(0.0625, 0.0625, 0.0625, 0.0625)); - const __m256d xi_153 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_150, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_151), - xi_152); - const __m256d xi_159 = _mm256_mul_pd( - xi_99, _mm256_set_pd(0.0416666666666667, 0.0416666666666667, - 0.0416666666666667, 0.0416666666666667)); - const __m256d xi_160 = _mm256_add_pd( - _mm256_mul_pd( - xi_125, _mm256_set_pd(0.0208333333333333, 0.0208333333333333, - 0.0208333333333333, 0.0208333333333333)), - xi_159); - const __m256d xi_161 = _mm256_add_pd( - _mm256_mul_pd(xi_158, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_160); - const __m256d xi_162 = _mm256_add_pd(xi_138, xi_161); - const __m256d xi_163 = - _mm256_add_pd(_mm256_add_pd(xi_150, xi_151), xi_152); - const __m256d xi_164 = _mm256_add_pd(xi_158, xi_160); - const __m256d xi_165 = _mm256_add_pd(xi_136, xi_164); - const __m256d xi_169 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_115), xi_104), - xi_17), - xi_209), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_171 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_126, xi_159), xi_168), - xi_169), - xi_170); - const __m256d xi_177 = _mm256_mul_pd( - xi_142, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d xi_178 = _mm256_add_pd(xi_176, xi_177); - const __m256d xi_179 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_173, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_175), - xi_178); - const __m256d xi_184 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_169, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_126), - xi_159), - xi_168), - xi_170); - const __m256d xi_185 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_129), xi_10), - xi_132), - xi_219), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_186 = - _mm256_mul_pd(xi_151, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_188 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_185, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_186), - xi_187); - const __m256d xi_189 = _mm256_add_pd(xi_148, xi_178); - const __m256d xi_192 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_190, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_161), - xi_191); - const __m256d xi_193 = - _mm256_add_pd(_mm256_add_pd(xi_185, xi_186), xi_187); - const __m256d xi_194 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_191, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_164), - xi_190); - const __m256d xi_195 = _mm256_add_pd( - _mm256_mul_pd(xi_177, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_176); - const __m256d xi_196 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_175, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_173), - xi_195); - const __m256d xi_197 = _mm256_add_pd(xi_146, xi_195); - const __m256d forceTerm_0 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_31, _mm256_set_pd(-1.5, -1.5, -1.5, -1.5)), - _mm256_mul_pd( - _mm256_mul_pd(xi_34, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))), - _mm256_mul_pd( - _mm256_mul_pd(xi_38, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))), - _mm256_mul_pd( - _mm256_mul_pd(xi_39, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))); - const __m256d forceTerm_1 = _mm256_add_pd(xi_40, xi_47); - const __m256d forceTerm_2 = _mm256_add_pd( - _mm256_mul_pd(xi_40, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_47); - const __m256d forceTerm_3 = _mm256_add_pd( - _mm256_mul_pd(xi_48, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_51); - const __m256d forceTerm_4 = _mm256_add_pd(xi_48, xi_51); - const __m256d forceTerm_5 = _mm256_add_pd(xi_52, xi_53); - const __m256d forceTerm_6 = _mm256_add_pd( - _mm256_mul_pd(xi_52, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), xi_53); - const __m256d forceTerm_7 = - _mm256_add_pd(_mm256_add_pd(xi_58, xi_60), xi_67); - const __m256d forceTerm_8 = - _mm256_add_pd(_mm256_add_pd(xi_57, xi_67), xi_68); - const __m256d forceTerm_9 = - _mm256_add_pd(_mm256_add_pd(xi_57, xi_60), xi_70); - const __m256d forceTerm_10 = - _mm256_add_pd(_mm256_add_pd(xi_58, xi_68), xi_70); - const __m256d forceTerm_11 = - _mm256_add_pd(_mm256_add_pd(xi_66, xi_71), xi_75); - const __m256d forceTerm_12 = - _mm256_add_pd(_mm256_add_pd(xi_69, xi_75), xi_76); - const __m256d forceTerm_13 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_78), xi_80); - const __m256d forceTerm_14 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_77), xi_80); - const __m256d forceTerm_15 = - _mm256_add_pd(_mm256_add_pd(xi_66, xi_76), xi_82); - const __m256d forceTerm_16 = - _mm256_add_pd(_mm256_add_pd(xi_69, xi_71), xi_82); - const __m256d forceTerm_17 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_77), xi_83); - const __m256d forceTerm_18 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_78), xi_83); - _mm256_store_pd( - &_data_pdfs_20_30_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_103, _mm256_set_pd(0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238)), - _mm256_mul_pd( - xi_86, _mm256_set_pd(0.1, 0.1, 0.1, 0.1))), - _mm256_mul_pd(xi_89, - _mm256_set_pd(0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429))), - _mm256_mul_pd(xi_99, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - forceTerm_0), - xi_212)); - _mm256_store_pd( - &_data_pdfs_20_31_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_107, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_1), - xi_114), - xi_119), - xi_128), - xi_215)); - _mm256_store_pd( - &_data_pdfs_20_32_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_119, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_2), - xi_107), - xi_113), - xi_128), - xi_213)); - _mm256_store_pd( - &_data_pdfs_20_33_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_131, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_3), - xi_134), - xi_136), - xi_137), - xi_200)); - _mm256_store_pd( - &_data_pdfs_20_34_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_134, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_4), - xi_131), - xi_137), - xi_138), - xi_207)); - _mm256_store_pd( - &_data_pdfs_20_35_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_141, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_5), - xi_143), - xi_146), - xi_147), - xi_210)); - _mm256_store_pd( - &_data_pdfs_20_36_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_143, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_6), - xi_141), - xi_147), - xi_148), - xi_217)); - _mm256_store_pd( - &_data_pdfs_20_37_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_7, xi_153), xi_157), - xi_162), - xi_211)); - _mm256_store_pd( - &_data_pdfs_20_38_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_8, xi_157), xi_163), - xi_165), - xi_205)); - _mm256_store_pd( - &_data_pdfs_20_39_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_9, xi_162), xi_163), - xi_167), - xi_216)); - _mm256_store_pd( - &_data_pdfs_20_310_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_10, xi_153), xi_165), - xi_167), - xi_204)); - _mm256_store_pd( - &_data_pdfs_20_311_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_11, xi_171), xi_179), - xi_182), - xi_203)); - _mm256_store_pd( - &_data_pdfs_20_312_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_12, xi_179), xi_183), - xi_184), - xi_206)); - _mm256_store_pd( - &_data_pdfs_20_313_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_13, xi_188), xi_189), - xi_192), - xi_201)); - _mm256_store_pd( - &_data_pdfs_20_314_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_14, xi_189), xi_193), - xi_194), - xi_208)); - _mm256_store_pd( - &_data_pdfs_20_315_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_15, xi_182), xi_184), - xi_196), - xi_209)); - _mm256_store_pd( - &_data_pdfs_20_316_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_16, xi_171), xi_183), - xi_196), - xi_202)); - _mm256_store_pd( - &_data_pdfs_20_317_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_17, xi_192), xi_193), - xi_197), - xi_218)); - _mm256_store_pd( - &_data_pdfs_20_318_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_18, xi_188), xi_194), - xi_197), - xi_219)); +static FUNC_PREFIX void kernel_collide(double * RESTRICT const _data_force, double * RESTRICT _data_pdfs, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, double omega_bulk, double omega_even, double omega_odd, double omega_shear) +{ + const double xi_50 = -omega_shear + 2.0; + const double xi_51 = xi_50*0.5; + const double xi_56 = xi_50*0.0833333333333333; + const double xi_61 = xi_50*0.166666666666667; + const double xi_71 = xi_50*0.25; + const double xi_76 = xi_50*0.0416666666666667; + const double xi_120 = omega_odd*0.25; + const double xi_126 = omega_odd*0.0833333333333333; + const double xi_159 = omega_shear*0.25; + const double xi_181 = omega_odd*0.0416666666666667; + const double xi_183 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_131 = rr_0*0.166666666666667; + const double xi_164 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_20_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_pdfs_20_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + 16*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_38_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_20_31_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_35; + double * RESTRICT _data_pdfs_20_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_318; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_20_311_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_311; + double * RESTRICT _data_pdfs_20_310_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_20_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_36; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_37_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_20_315_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_315; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_pdfs_20_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_314; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_39_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_313; + double * RESTRICT _data_pdfs_20_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_317; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_pdfs_20_32_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_20_312_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_312; + double * RESTRICT _data_pdfs_20_316_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_316; + for (int64_t ctr_0 = 0; ctr_0 < ((_size_force_0) % (4) == 0 ? _size_force_0 : ((int64_t)((_size_force_0) / (4))+1) * (4)); ctr_0 += 4) + { + const __m256d xi_207 = _mm256_load_pd(& _data_pdfs_20_38_10[ctr_0]); + const __m256d xi_208 = _mm256_load_pd(& _data_pdfs_20_31_10[ctr_0]); + const __m256d xi_209 = _mm256_load_pd(& _data_pdfs_20_35_10[ctr_0]); + const __m256d xi_210 = _mm256_load_pd(& _data_pdfs_20_318_10[ctr_0]); + const __m256d xi_211 = _mm256_load_pd(& _data_pdfs_20_30_10[ctr_0]); + const __m256d xi_212 = _mm256_load_pd(& _data_pdfs_20_311_10[ctr_0]); + const __m256d xi_213 = _mm256_load_pd(& _data_pdfs_20_310_10[ctr_0]); + const __m256d xi_214 = _mm256_load_pd(& _data_pdfs_20_34_10[ctr_0]); + const __m256d xi_215 = _mm256_load_pd(& _data_pdfs_20_36_10[ctr_0]); + const __m256d xi_216 = _mm256_load_pd(& _data_pdfs_20_33_10[ctr_0]); + const __m256d xi_217 = _mm256_load_pd(& _data_pdfs_20_37_10[ctr_0]); + const __m256d xi_218 = _mm256_load_pd(& _data_pdfs_20_315_10[ctr_0]); + const __m256d xi_219 = _mm256_load_pd(& _data_force_20_31_10[ctr_0]); + const __m256d xi_220 = _mm256_load_pd(& _data_pdfs_20_314_10[ctr_0]); + const __m256d xi_221 = _mm256_load_pd(& _data_force_20_32_10[ctr_0]); + const __m256d xi_222 = _mm256_load_pd(& _data_pdfs_20_39_10[ctr_0]); + const __m256d xi_223 = _mm256_load_pd(& _data_pdfs_20_313_10[ctr_0]); + const __m256d xi_224 = _mm256_load_pd(& _data_pdfs_20_317_10[ctr_0]); + const __m256d xi_225 = _mm256_load_pd(& _data_force_20_30_10[ctr_0]); + const __m256d xi_226 = _mm256_load_pd(& _data_pdfs_20_32_10[ctr_0]); + const __m256d xi_227 = _mm256_load_pd(& _data_pdfs_20_312_10[ctr_0]); + const __m256d xi_228 = _mm256_load_pd(& _data_pdfs_20_316_10[ctr_0]); + const __m256d xi_0 = _mm256_add_pd(xi_207,xi_213); + const __m256d xi_1 = _mm256_add_pd(xi_210,xi_220); + const __m256d xi_2 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_1),xi_214); + const __m256d xi_3 = _mm256_add_pd(xi_212,xi_218); + const __m256d xi_4 = _mm256_add_pd(xi_208,xi_3); + const __m256d xi_5 = _mm256_add_pd(xi_217,xi_4); + const __m256d xi_6 = _mm256_add_pd(xi_209,xi_227); + const __m256d xi_7 = _mm256_add_pd(xi_223,xi_6); + const __m256d xi_8 = _mm256_add_pd(xi_216,xi_222); + const __m256d xi_9 = _mm256_add_pd(xi_226,xi_228); + const __m256d xi_10 = _mm256_add_pd(xi_215,xi_224); + const __m256d xi_12 = _mm256_mul_pd(xi_224,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_13 = _mm256_mul_pd(xi_216,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_14 = _mm256_add_pd(xi_12,xi_13); + const __m256d xi_15 = _mm256_mul_pd(xi_223,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_16 = _mm256_mul_pd(xi_217,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_17 = _mm256_mul_pd(xi_222,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_18 = _mm256_add_pd(xi_16,xi_17); + const __m256d xi_19 = _mm256_add_pd(xi_15,xi_18); + const __m256d xi_20 = _mm256_mul_pd(xi_213,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_21 = _mm256_add_pd(xi_20,xi_207); + const __m256d xi_22 = _mm256_mul_pd(xi_227,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_23 = _mm256_mul_pd(xi_226,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_24 = _mm256_mul_pd(xi_228,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_25 = _mm256_add_pd(_mm256_add_pd(xi_22,xi_23),xi_24); + const __m256d xi_26 = _mm256_mul_pd(xi_210,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_27 = _mm256_add_pd(xi_12,xi_26); + const __m256d xi_28 = _mm256_mul_pd(xi_218,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_29 = _mm256_mul_pd(xi_215,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_30 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_212,xi_24),xi_28),xi_29); + const __m256d xi_31 = _mm256_add_pd(xi_223,xi_224); + const __m256d xi_32 = _mm256_add_pd(_mm256_add_pd(xi_217,xi_31),xi_8); + const __m256d xi_33 = _mm256_add_pd(_mm256_add_pd(xi_16,xi_21),xi_222); + const __m256d xi_34 = _mm256_add_pd(xi_224,xi_26); + const __m256d xi_35 = _mm256_add_pd(xi_15,xi_220); + const __m256d xi_36 = _mm256_add_pd(xi_34,xi_35); + const __m256d xi_37 = _mm256_add_pd(_mm256_add_pd(xi_222,xi_227),xi_9); + const __m256d xi_38 = _mm256_add_pd(xi_228,xi_28); + const __m256d xi_39 = _mm256_add_pd(_mm256_add_pd(xi_212,xi_22),xi_38); + const __m256d xi_55 = _mm256_mul_pd(xi_219,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_63 = _mm256_mul_pd(xi_225,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_67 = _mm256_mul_pd(xi_221,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_70 = _mm256_mul_pd(xi_219,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_74 = _mm256_mul_pd(xi_225,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_78 = _mm256_mul_pd(xi_219,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_88 = _mm256_mul_pd(xi_221,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_99 = _mm256_mul_pd(xi_211,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_100 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_209,_mm256_set_pd(3.0,3.0,3.0,3.0)),_mm256_mul_pd(xi_215,_mm256_set_pd(3.0,3.0,3.0,3.0))),xi_99); + const __m256d xi_101 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_208,_mm256_set_pd(3.0,3.0,3.0,3.0)),_mm256_mul_pd(xi_212,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),_mm256_mul_pd(xi_218,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),_mm256_mul_pd(xi_226,_mm256_set_pd(3.0,3.0,3.0,3.0))),_mm256_mul_pd(xi_227,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),_mm256_mul_pd(xi_228,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),xi_100),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_102 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_212,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_218,_mm256_set_pd(2.0,2.0,2.0,2.0))),_mm256_mul_pd(xi_227,_mm256_set_pd(2.0,2.0,2.0,2.0))),_mm256_mul_pd(xi_228,_mm256_set_pd(2.0,2.0,2.0,2.0))); + const __m256d xi_103 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_214,_mm256_set_pd(5.0,5.0,5.0,5.0)),_mm256_mul_pd(xi_216,_mm256_set_pd(5.0,5.0,5.0,5.0))),xi_102); + const __m256d xi_104 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_208,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0)),_mm256_mul_pd(xi_210,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_220,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_223,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_224,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_226,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0))),xi_100),xi_103),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_107 = _mm256_mul_pd(xi_212,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_108 = _mm256_add_pd(xi_107,xi_22); + const __m256d xi_109 = _mm256_mul_pd(xi_207,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_112 = _mm256_mul_pd(xi_220,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_113 = _mm256_add_pd(_mm256_add_pd(xi_112,xi_19),xi_27); + const __m256d xi_115 = _mm256_mul_pd(xi_223,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_116 = _mm256_mul_pd(xi_220,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_117 = _mm256_add_pd(_mm256_mul_pd(xi_210,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_224,_mm256_set_pd(2.0,2.0,2.0,2.0))); + const __m256d xi_118 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_207,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0)),_mm256_mul_pd(xi_208,_mm256_set_pd(5.0,5.0,5.0,5.0))),_mm256_mul_pd(xi_209,_mm256_set_pd(-4.0,-4.0,-4.0,-4.0))),_mm256_mul_pd(xi_213,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0))),_mm256_mul_pd(xi_215,_mm256_set_pd(-4.0,-4.0,-4.0,-4.0))),_mm256_mul_pd(xi_217,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0))),_mm256_mul_pd(xi_222,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0))),_mm256_mul_pd(xi_226,_mm256_set_pd(5.0,5.0,5.0,5.0))),xi_103),xi_115),xi_116),xi_117),xi_99),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_119 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_107,xi_208),xi_227),xi_23),xi_38); + const __m256d xi_121 = _mm256_mul_pd(xi_119,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_122 = _mm256_mul_pd(xi_217,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_123 = _mm256_mul_pd(xi_213,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_124 = _mm256_add_pd(_mm256_mul_pd(xi_207,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0)),_mm256_mul_pd(xi_222,_mm256_set_pd(2.0,2.0,2.0,2.0))); + const __m256d xi_125 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_122,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_123),xi_124),xi_25),xi_4); + const __m256d xi_127 = _mm256_mul_pd(xi_125,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_128 = _mm256_mul_pd(xi_127,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_133 = _mm256_mul_pd(xi_118,_mm256_set_pd(-0.0198412698412698,-0.0198412698412698,-0.0198412698412698,-0.0198412698412698)); + const __m256d xi_144 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_112,xi_13),xi_214),xi_223),xi_34); + const __m256d xi_145 = _mm256_mul_pd(xi_144,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_146 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_123,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_122),xi_124),xi_14),xi_210),xi_214),xi_35); + const __m256d xi_147 = _mm256_mul_pd(xi_146,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_149 = _mm256_mul_pd(xi_147,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_150 = _mm256_add_pd(xi_218,xi_228); + const __m256d xi_151 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_108,xi_150),xi_209),xi_29); + const __m256d xi_152 = _mm256_mul_pd(xi_151,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_153 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_115,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_116,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_117),xi_30),xi_6); + const __m256d xi_154 = _mm256_mul_pd(xi_153,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_155 = _mm256_mul_pd(xi_154,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_162 = _mm256_mul_pd(xi_118,_mm256_set_pd(0.0138888888888889,0.0138888888888889,0.0138888888888889,0.0138888888888889)); + const __m256d xi_177 = _mm256_mul_pd(xi_104,_mm256_set_pd(-0.00714285714285714,-0.00714285714285714,-0.00714285714285714,-0.00714285714285714)); + const __m256d xi_179 = _mm256_mul_pd(xi_101,_mm256_set_pd(0.025,0.025,0.025,0.025)); + const __m256d xi_182 = _mm256_mul_pd(xi_153,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d xi_184 = _mm256_mul_pd(xi_151,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_185 = _mm256_mul_pd(xi_118,_mm256_set_pd(-0.00396825396825397,-0.00396825396825397,-0.00396825396825397,-0.00396825396825397)); + const __m256d xi_189 = _mm256_mul_pd(xi_125,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d xi_190 = _mm256_mul_pd(xi_119,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_196 = _mm256_mul_pd(xi_104,_mm256_set_pd(0.0178571428571429,0.0178571428571429,0.0178571428571429,0.0178571428571429)); + const __m256d xi_199 = _mm256_mul_pd(xi_144,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_200 = _mm256_mul_pd(xi_146,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d vel0Term = xi_2; + const __m256d vel1Term = xi_5; + const __m256d vel2Term = xi_7; + const __m256d rho = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel0Term,vel1Term),vel2Term),xi_10),xi_211),xi_8),xi_9); + const __m256d xi_11 = _mm256_div_pd(_mm256_set_pd(1.0,1.0,1.0,1.0),rho); + const __m256d u_0 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(vel0Term,xi_14),xi_19)); + const __m256d xi_40 = _mm256_mul_pd(u_0,xi_225); + const __m256d xi_41 = _mm256_mul_pd(xi_40,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_47 = _mm256_mul_pd(xi_41,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_105 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_0,u_0))); + const __m256d xi_141 = _mm256_mul_pd(rho,u_0); + const __m256d xi_142 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel0Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_141),xi_32); + const __m256d xi_143 = _mm256_mul_pd(xi_142,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_167 = _mm256_mul_pd(xi_142,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d u_1 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel1Term,xi_17),xi_21),xi_25)); + const __m256d xi_42 = _mm256_mul_pd(u_1,xi_219); + const __m256d xi_43 = _mm256_mul_pd(xi_42,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_48 = _mm256_mul_pd(xi_43,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_69 = _mm256_mul_pd(u_1,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_72 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_0,xi_70),_mm256_mul_pd(xi_225,xi_69)),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_73 = _mm256_mul_pd(xi_72,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_110 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_1,u_1))); + const __m256d xi_111 = _mm256_add_pd(_mm256_add_pd(xi_109,xi_110),xi_20); + const __m256d xi_129 = _mm256_mul_pd(rho,u_1); + const __m256d xi_130 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel1Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_109),xi_129),xi_213),xi_37); + const __m256d xi_132 = _mm256_mul_pd(xi_130,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_165 = _mm256_mul_pd(xi_130,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d xi_166 = _mm256_add_pd(xi_127,xi_165); + const __m256d xi_175 = _mm256_mul_pd(xi_165,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_176 = _mm256_add_pd(xi_128,xi_175); + const __m256d xi_191 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_189,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_165),xi_190); + const __m256d xi_192 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_190,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_175),xi_189); + const __m256d u_2 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel2Term,xi_220),xi_27),xi_30)); + const __m256d xi_44 = _mm256_mul_pd(u_2,xi_221); + const __m256d xi_45 = _mm256_mul_pd(xi_44,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_46 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(xi_41,xi_43),xi_45),_mm256_set_pd(-omega_bulk + 2.0,-omega_bulk + 2.0,-omega_bulk + 2.0,-omega_bulk + 2.0)); + const __m256d xi_49 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_44,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_47),xi_48); + const __m256d xi_52 = _mm256_mul_pd(xi_45,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_53 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_42,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_47),xi_52); + const __m256d xi_54 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_40,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_48),xi_52); + const __m256d xi_57 = _mm256_mul_pd(xi_49,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_58 = _mm256_mul_pd(xi_57,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_59 = _mm256_mul_pd(xi_54,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_60 = _mm256_mul_pd(xi_59,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_62 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_58),xi_60); + const __m256d xi_64 = _mm256_mul_pd(xi_53,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_65 = _mm256_mul_pd(xi_64,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_66 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_58),xi_65); + const __m256d xi_68 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_60),xi_65); + const __m256d xi_75 = _mm256_add_pd(_mm256_mul_pd(xi_74,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_59); + const __m256d xi_77 = _mm256_mul_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_79 = _mm256_mul_pd(xi_46,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_80 = _mm256_add_pd(xi_64,xi_79); + const __m256d xi_81 = _mm256_add_pd(xi_78,xi_80); + const __m256d xi_82 = _mm256_add_pd(xi_77,xi_81); + const __m256d xi_83 = _mm256_add_pd(xi_59,xi_74); + const __m256d xi_84 = _mm256_add_pd(_mm256_mul_pd(xi_78,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_80); + const __m256d xi_85 = _mm256_add_pd(xi_77,xi_84); + const __m256d xi_86 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_2,xi_70),_mm256_mul_pd(xi_221,xi_69)),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_87 = _mm256_mul_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_89 = _mm256_add_pd(xi_57,xi_88); + const __m256d xi_90 = _mm256_add_pd(xi_87,xi_89); + const __m256d xi_91 = _mm256_mul_pd(xi_86,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_92 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_mul_pd(u_0,xi_221),_mm256_set_pd(0.5,0.5,0.5,0.5)),_mm256_mul_pd(_mm256_mul_pd(u_2,xi_225),_mm256_set_pd(0.5,0.5,0.5,0.5))),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_93 = _mm256_mul_pd(xi_92,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_94 = _mm256_mul_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_95 = _mm256_add_pd(_mm256_add_pd(xi_79,xi_89),xi_94); + const __m256d xi_96 = _mm256_add_pd(_mm256_mul_pd(xi_88,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_57); + const __m256d xi_97 = _mm256_add_pd(xi_87,xi_96); + const __m256d xi_98 = _mm256_add_pd(_mm256_add_pd(xi_79,xi_94),xi_96); + const __m256d xi_106 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_2,u_2))); + const __m256d xi_114 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_105,xi_106),xi_108),xi_111),xi_113),xi_211),xi_24),xi_28),_mm256_set_pd(omega_bulk,omega_bulk,omega_bulk,omega_bulk)); + const __m256d xi_134 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_106,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_209),xi_215); + const __m256d xi_135 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_208,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_1),xi_111),xi_134),xi_18),xi_23),xi_31),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_136 = _mm256_mul_pd(xi_135,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_137 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_105,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_110,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),_mm256_mul_pd(xi_214,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0))),_mm256_mul_pd(xi_216,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0))),xi_102),xi_109),xi_113),xi_134),xi_20),xi_208),xi_226),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_138 = _mm256_mul_pd(xi_137,_mm256_set_pd(-0.0416666666666667,-0.0416666666666667,-0.0416666666666667,-0.0416666666666667)); + const __m256d xi_139 = _mm256_add_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-0.05,-0.05,-0.05,-0.05)),xi_138); + const __m256d xi_140 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(0.0142857142857143,0.0142857142857143,0.0142857142857143,0.0142857142857143)),xi_133),xi_136),xi_139); + const __m256d xi_148 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(-0.0357142857142857,-0.0357142857142857,-0.0357142857142857,-0.0357142857142857)),_mm256_mul_pd(xi_137,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333))),xi_133); + const __m256d xi_156 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(rho,u_2),_mm256_mul_pd(vel2Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_10),xi_107),xi_112),xi_150),xi_210); + const __m256d xi_157 = _mm256_mul_pd(xi_156,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_158 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(-0.0214285714285714,-0.0214285714285714,-0.0214285714285714,-0.0214285714285714)),_mm256_mul_pd(xi_118,_mm256_set_pd(0.0158730158730159,0.0158730158730159,0.0158730158730159,0.0158730158730159))),_mm256_mul_pd(xi_136,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_139); + const __m256d xi_161 = _mm256_mul_pd(xi_135,_mm256_set_pd(0.0625,0.0625,0.0625,0.0625)); + const __m256d xi_168 = _mm256_mul_pd(xi_114,_mm256_set_pd(0.0416666666666667,0.0416666666666667,0.0416666666666667,0.0416666666666667)); + const __m256d xi_169 = _mm256_add_pd(_mm256_mul_pd(xi_137,_mm256_set_pd(0.0208333333333333,0.0208333333333333,0.0208333333333333,0.0208333333333333)),xi_168); + const __m256d xi_170 = _mm256_add_pd(_mm256_mul_pd(xi_167,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_169); + const __m256d xi_171 = _mm256_add_pd(xi_149,xi_170); + const __m256d xi_173 = _mm256_add_pd(xi_167,xi_169); + const __m256d xi_174 = _mm256_add_pd(xi_147,xi_173); + const __m256d xi_186 = _mm256_mul_pd(xi_156,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d xi_187 = _mm256_add_pd(xi_185,xi_186); + const __m256d xi_188 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_182,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_184),xi_187); + const __m256d xi_195 = _mm256_mul_pd(xi_161,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_198 = _mm256_add_pd(xi_154,xi_187); + const __m256d xi_201 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_199,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_170),xi_200); + const __m256d xi_203 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_200,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_173),xi_199); + const __m256d xi_204 = _mm256_add_pd(_mm256_mul_pd(xi_186,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_185); + const __m256d xi_205 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_184,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_182),xi_204); + const __m256d xi_206 = _mm256_add_pd(xi_155,xi_204); + const __m256d p_0 = _mm256_add_pd(xi_2,xi_32); + const __m256d p_1 = xi_33; + const __m256d xi_160 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_1,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_0,xi_129)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_163 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_160,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_161),xi_162); + const __m256d xi_172 = _mm256_add_pd(_mm256_add_pd(xi_160,xi_161),xi_162); + const __m256d p_2 = xi_36; + const __m256d xi_194 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_2,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_141)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_197 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_194,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_195),xi_196); + const __m256d xi_202 = _mm256_add_pd(_mm256_add_pd(xi_194,xi_195),xi_196); + const __m256d p_3 = xi_33; + const __m256d p_4 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_37),xi_5); + const __m256d p_5 = xi_39; + const __m256d xi_178 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_5,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_129)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_180 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_138,xi_168),xi_177),xi_178),xi_179); + const __m256d xi_193 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_178,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_138),xi_168),xi_177),xi_179); + const __m256d p_6 = xi_36; + const __m256d p_7 = xi_39; + const __m256d p_8 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_1,xi_10),xi_228),xi_3),xi_7); + const __m256d forceTerm_0 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_46,_mm256_set_pd(-1.5,-1.5,-1.5,-1.5)),_mm256_mul_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))),_mm256_mul_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))),_mm256_mul_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))); + const __m256d forceTerm_1 = _mm256_add_pd(xi_55,xi_62); + const __m256d forceTerm_2 = _mm256_add_pd(_mm256_mul_pd(xi_55,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_62); + const __m256d forceTerm_3 = _mm256_add_pd(_mm256_mul_pd(xi_63,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_66); + const __m256d forceTerm_4 = _mm256_add_pd(xi_63,xi_66); + const __m256d forceTerm_5 = _mm256_add_pd(xi_67,xi_68); + const __m256d forceTerm_6 = _mm256_add_pd(_mm256_mul_pd(xi_67,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_68); + const __m256d forceTerm_7 = _mm256_add_pd(_mm256_add_pd(xi_73,xi_75),xi_82); + const __m256d forceTerm_8 = _mm256_add_pd(_mm256_add_pd(xi_72,xi_82),xi_83); + const __m256d forceTerm_9 = _mm256_add_pd(_mm256_add_pd(xi_72,xi_75),xi_85); + const __m256d forceTerm_10 = _mm256_add_pd(_mm256_add_pd(xi_73,xi_83),xi_85); + const __m256d forceTerm_11 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_86),xi_90); + const __m256d forceTerm_12 = _mm256_add_pd(_mm256_add_pd(xi_84,xi_90),xi_91); + const __m256d forceTerm_13 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_93),xi_95); + const __m256d forceTerm_14 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_92),xi_95); + const __m256d forceTerm_15 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_91),xi_97); + const __m256d forceTerm_16 = _mm256_add_pd(_mm256_add_pd(xi_84,xi_86),xi_97); + const __m256d forceTerm_17 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_92),xi_98); + const __m256d forceTerm_18 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_93),xi_98); + _mm256_store_pd(&_data_pdfs_20_30_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(0.1,0.1,0.1,0.1)),_mm256_mul_pd(xi_104,_mm256_set_pd(0.0428571428571429,0.0428571428571429,0.0428571428571429,0.0428571428571429))),_mm256_mul_pd(xi_114,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_mul_pd(xi_118,_mm256_set_pd(0.0238095238095238,0.0238095238095238,0.0238095238095238,0.0238095238095238))),forceTerm_0),xi_211)); + _mm256_store_pd(&_data_pdfs_20_31_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_121,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_1),xi_128),xi_132),xi_140),xi_208)); + _mm256_store_pd(&_data_pdfs_20_32_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_132,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_2),xi_121),xi_127),xi_140),xi_226)); + _mm256_store_pd(&_data_pdfs_20_33_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_143,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_3),xi_145),xi_147),xi_148),xi_216)); + _mm256_store_pd(&_data_pdfs_20_34_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_145,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_4),xi_143),xi_148),xi_149),xi_214)); + _mm256_store_pd(&_data_pdfs_20_35_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_152,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_5),xi_155),xi_157),xi_158),xi_209)); + _mm256_store_pd(&_data_pdfs_20_36_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_157,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_6),xi_152),xi_154),xi_158),xi_215)); + _mm256_store_pd(&_data_pdfs_20_37_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_7,xi_163),xi_166),xi_171),xi_217)); + _mm256_store_pd(&_data_pdfs_20_38_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_8,xi_166),xi_172),xi_174),xi_207)); + _mm256_store_pd(&_data_pdfs_20_39_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_9,xi_171),xi_172),xi_176),xi_222)); + _mm256_store_pd(&_data_pdfs_20_310_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_10,xi_163),xi_174),xi_176),xi_213)); + _mm256_store_pd(&_data_pdfs_20_311_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_11,xi_180),xi_188),xi_191),xi_212)); + _mm256_store_pd(&_data_pdfs_20_312_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_12,xi_188),xi_192),xi_193),xi_227)); + _mm256_store_pd(&_data_pdfs_20_313_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_13,xi_197),xi_198),xi_201),xi_223)); + _mm256_store_pd(&_data_pdfs_20_314_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_14,xi_198),xi_202),xi_203),xi_220)); + _mm256_store_pd(&_data_pdfs_20_315_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_15,xi_191),xi_193),xi_205),xi_218)); + _mm256_store_pd(&_data_pdfs_20_316_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_16,xi_180),xi_192),xi_205),xi_228)); + _mm256_store_pd(&_data_pdfs_20_317_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_17,xi_201),xi_202),xi_206),xi_224)); + _mm256_store_pd(&_data_pdfs_20_318_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_18,xi_197),xi_203),xi_206),xi_210)); + } } - } - } + } +} } -} // namespace internal_kernel_collide namespace internal_kernel_stream { -static FUNC_PREFIX void -kernel_stream(double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, - int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_1, - int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3) { - for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - for (int64_t ctr_0 = 1; - ctr_0 < ((_size_pdfs_0 - 2) % (4) == 0 - ? _size_pdfs_0 - 2 - : ((int64_t)((_size_pdfs_0 - 2) / (4)) + 1) * (4)) + - 1; - ctr_0 += 4) { - _mm256_store_pd(&_data_pdfs_tmp_20_30_10[ctr_0], - _mm256_load_pd(&_data_pdfs_20_30_10[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_31_10[ctr_0], - _mm256_load_pd(&_data_pdfs_20_31_1m1[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_32_10[ctr_0], - _mm256_load_pd(&_data_pdfs_20_32_11[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_33_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_34_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_35_10[ctr_0], - _mm256_load_pd(&_data_pdfs_2m1_35_10[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_36_10[ctr_0], - _mm256_load_pd(&_data_pdfs_21_36_10[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_37_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_38_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_39_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_310_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_311_10[ctr_0], - _mm256_load_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_312_10[ctr_0], - _mm256_load_pd(&_data_pdfs_2m1_312_11[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_313_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_314_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_315_10[ctr_0], - _mm256_load_pd(&_data_pdfs_21_315_1m1[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_316_10[ctr_0], - _mm256_load_pd(&_data_pdfs_21_316_11[ctr_0])); - _mm256_store_pd(&_data_pdfs_tmp_20_317_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])); - _mm256_store_pd(&_data_pdfs_tmp_20_318_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])); +static FUNC_PREFIX void kernel_stream(double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3) +{ + for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + for (int64_t ctr_0 = 1; ctr_0 < ((_size_pdfs_0 - 2) % (4) == 0 ? _size_pdfs_0 - 2 : ((int64_t)((_size_pdfs_0 - 2) / (4))+1) * (4)) + 1; ctr_0 += 4) + { + _mm256_store_pd(&_data_pdfs_tmp_20_30_10[ctr_0],_mm256_load_pd(& _data_pdfs_20_30_10[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_31_10[ctr_0],_mm256_load_pd(& _data_pdfs_20_31_1m1[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_32_10[ctr_0],_mm256_load_pd(& _data_pdfs_20_32_11[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_33_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_34_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_35_10[ctr_0],_mm256_load_pd(& _data_pdfs_2m1_35_10[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_36_10[ctr_0],_mm256_load_pd(& _data_pdfs_21_36_10[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_37_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_38_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_39_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_310_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_311_10[ctr_0],_mm256_load_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_312_10[ctr_0],_mm256_load_pd(& _data_pdfs_2m1_312_11[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_313_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_314_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_315_10[ctr_0],_mm256_load_pd(& _data_pdfs_21_315_1m1[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_316_10[ctr_0],_mm256_load_pd(& _data_pdfs_21_316_11[ctr_0])); + _mm256_store_pd(&_data_pdfs_tmp_20_317_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])); + _mm256_store_pd(&_data_pdfs_tmp_20_318_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + } } - } - } + } +} } -} // namespace internal_kernel_stream -const real_t MRTLatticeModelAvx::w[19] = { - 0.333333333333333, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778}; -const real_t MRTLatticeModelAvx::wInv[19] = { - 3.00000000000000, 18.0000000000000, 18.0000000000000, 18.0000000000000, - 18.0000000000000, 18.0000000000000, 18.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000}; -void MRTLatticeModelAvx::Sweep::streamCollide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); +const real_t MRTLatticeModelAvx::w[19] = { 0.333333333333333,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778 }; +const real_t MRTLatticeModelAvx::wInv[19] = { 3.00000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000 }; + +void MRTLatticeModelAvx::Sweep::streamCollide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - auto &lm = - dynamic_cast *>(pdfs)->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &force = lm.force_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &omega_odd = lm.omega_odd_; - auto &omega_bulk = lm.omega_bulk_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_streamCollide::kernel_streamCollide( - _data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, - _size_force_2, _stride_force_1, _stride_force_2, _stride_force_3, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, omega_bulk, omega_even, omega_odd, - omega_shear); - pdfs->swapDataPointers(pdfs_tmp); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & force = lm.force_; + auto & omega_even = lm.omega_even_; + auto & omega_odd = lm.omega_odd_; + auto & omega_shear = lm.omega_shear_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_streamCollide::kernel_streamCollide(_data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, _size_force_2, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, omega_bulk, omega_even, omega_odd, omega_shear); + pdfs->swapDataPointers(pdfs_tmp); + } -void MRTLatticeModelAvx::Sweep::collide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); +void MRTLatticeModelAvx::Sweep::collide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); - auto &lm = - dynamic_cast *>(pdfs)->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &force = lm.force_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &omega_odd = lm.omega_odd_; - auto &omega_bulk = lm.omega_bulk_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_kernel_collide::kernel_collide( - _data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, - _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_1, - _stride_pdfs_2, _stride_pdfs_3, omega_bulk, omega_even, omega_odd, - omega_shear); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & force = lm.force_; + auto & omega_even = lm.omega_even_; + auto & omega_odd = lm.omega_odd_; + auto & omega_shear = lm.omega_shear_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_kernel_collide::kernel_collide(_data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, omega_bulk, omega_even, omega_odd, omega_shear); } -void MRTLatticeModelAvx::Sweep::stream( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); + +void MRTLatticeModelAvx::Sweep::stream( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_0 = - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_1 = - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_2 = - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_stream::kernel_stream( - _data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); - pdfs->swapDataPointers(pdfs_tmp); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_stream::kernel_stream(_data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); + + pdfs->swapDataPointers(pdfs_tmp); + } + } // namespace lbm } // namespace walberla + + + // Buffer Packing namespace walberla { namespace mpi { -mpi::SendBuffer &operator<<(mpi::SendBuffer &buf, - const ::walberla::lbm::MRTLatticeModelAvx &lm) { - buf << lm.currentLevel; - return buf; +mpi::SendBuffer & operator<< (mpi::SendBuffer & buf, const ::walberla::lbm::MRTLatticeModelAvx & lm) +{ + buf << lm.currentLevel; + return buf; } -mpi::RecvBuffer &operator>>(mpi::RecvBuffer &buf, - ::walberla::lbm::MRTLatticeModelAvx &lm) { - buf >> lm.currentLevel; - return buf; +mpi::RecvBuffer & operator>> (mpi::RecvBuffer & buf, ::walberla::lbm::MRTLatticeModelAvx & lm) +{ + buf >> lm.currentLevel; + return buf; } + } // namespace mpi } // namespace walberla -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic pop +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic pop #endif \ No newline at end of file diff --git a/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.h b/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.h index f7e428c7c4d..0d287bdaa6b 100644 --- a/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.h +++ b/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.h @@ -687,16 +687,101 @@ template<> struct PressureTensor { template< typename FieldPtrOrIterator > - static void get( Matrix3< real_t > & /* pressureTensor */, const MRTLatticeModelAvx & /* latticeModel */, const FieldPtrOrIterator & /* it */ ) + static void get( Matrix3< real_t > & pressureTensor, const MRTLatticeModelAvx & lm, const FieldPtrOrIterator & it ) { - WALBERLA_ABORT("Not implemented"); + const auto x = it.x(); + const auto y = it.y(); + const auto z = it.z(); + + const real_t f_0 = it[0]; + const real_t f_1 = it[1]; + const real_t f_2 = it[2]; + const real_t f_3 = it[3]; + const real_t f_4 = it[4]; + const real_t f_5 = it[5]; + const real_t f_6 = it[6]; + const real_t f_7 = it[7]; + const real_t f_8 = it[8]; + const real_t f_9 = it[9]; + const real_t f_10 = it[10]; + const real_t f_11 = it[11]; + const real_t f_12 = it[12]; + const real_t f_13 = it[13]; + const real_t f_14 = it[14]; + const real_t f_15 = it[15]; + const real_t f_16 = it[16]; + const real_t f_17 = it[17]; + const real_t f_18 = it[18]; + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } template< typename PdfField_T > - static void get( Matrix3< real_t > & /* pressureTensor */, const MRTLatticeModelAvx & /* latticeModel */, const PdfField_T & /* pdf */, - const cell_idx_t /* x */, const cell_idx_t /* y */, const cell_idx_t /* z */ ) + static void get( Matrix3< real_t > & pressureTensor , const MRTLatticeModelAvx & lm , const PdfField_T & pdf, + const cell_idx_t x, const cell_idx_t y, const cell_idx_t z ) { - WALBERLA_ABORT("Not implemented"); + const real_t & xyz0 = pdf(x,y,z,0); + const real_t f_0 = pdf.getF( &xyz0, 0); + const real_t f_1 = pdf.getF( &xyz0, 1); + const real_t f_2 = pdf.getF( &xyz0, 2); + const real_t f_3 = pdf.getF( &xyz0, 3); + const real_t f_4 = pdf.getF( &xyz0, 4); + const real_t f_5 = pdf.getF( &xyz0, 5); + const real_t f_6 = pdf.getF( &xyz0, 6); + const real_t f_7 = pdf.getF( &xyz0, 7); + const real_t f_8 = pdf.getF( &xyz0, 8); + const real_t f_9 = pdf.getF( &xyz0, 9); + const real_t f_10 = pdf.getF( &xyz0, 10); + const real_t f_11 = pdf.getF( &xyz0, 11); + const real_t f_12 = pdf.getF( &xyz0, 12); + const real_t f_13 = pdf.getF( &xyz0, 13); + const real_t f_14 = pdf.getF( &xyz0, 14); + const real_t f_15 = pdf.getF( &xyz0, 15); + const real_t f_16 = pdf.getF( &xyz0, 16); + const real_t f_17 = pdf.getF( &xyz0, 17); + const real_t f_18 = pdf.getF( &xyz0, 18); + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } }; diff --git a/src/walberla_bridge/generated_kernels/PushPackInfo.cpp b/src/walberla_bridge/generated_kernels/PushPackInfo.cpp index ce20e466e48..429b5cb4fe0 100644 --- a/src/walberla_bridge/generated_kernels/PushPackInfo.cpp +++ b/src/walberla_bridge/generated_kernels/PushPackInfo.cpp @@ -1,2240 +1,1705 @@ -#include "PushPackInfo.h" -#include "core/DataTypes.h" -#include "core/cell/CellInterval.h" #include "stencil/Directions.h" +#include "core/cell/CellInterval.h" +#include "core/DataTypes.h" +#include "PushPackInfo.h" -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#pragma GCC diagnostic ignored "-Wshadow" -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wunused-variable" +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wfloat-equal" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wunused-variable" #endif + + namespace walberla { namespace lbm { using walberla::cell::CellInterval; using walberla::stencil::Direction; + + namespace internal_pack_SW { -static FUNC_PREFIX void -pack_SW(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void pack_SW(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_SW namespace internal_pack_BW { -static FUNC_PREFIX void -pack_BW(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void pack_BW(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_BW namespace internal_pack_W { -static FUNC_PREFIX void -pack_W(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void pack_W(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_W namespace internal_pack_TW { -static FUNC_PREFIX void -pack_TW(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void pack_TW(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_TW namespace internal_pack_NW { -static FUNC_PREFIX void -pack_NW(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void pack_NW(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_NW namespace internal_pack_BS { -static FUNC_PREFIX void -pack_BS(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0]; +static FUNC_PREFIX void pack_BS(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0]; + } } - } - } + } +} } -} // namespace internal_pack_BS namespace internal_pack_S { -static FUNC_PREFIX void -pack_S(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - double *RESTRICT _data_pdfs_20_32_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_20_32_1m1[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_S(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + double * RESTRICT _data_pdfs_20_32_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_20_32_1m1[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_S namespace internal_pack_TS { -static FUNC_PREFIX void -pack_TS(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0]; +static FUNC_PREFIX void pack_TS(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0]; + } } - } - } + } +} } -} // namespace internal_pack_TS namespace internal_pack_B { -static FUNC_PREFIX void -pack_B(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - double *RESTRICT _data_pdfs_2m1_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_36; - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_2m1_36_10[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_B(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_36 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + double * RESTRICT _data_pdfs_2m1_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_36; + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_2m1_36_10[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_B namespace internal_pack_T { -static FUNC_PREFIX void -pack_T(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - double *RESTRICT _data_pdfs_21_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_35; - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_21_35_10[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_T(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + double * RESTRICT _data_pdfs_21_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_35; + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_21_35_10[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_T namespace internal_pack_BN { -static FUNC_PREFIX void -pack_BN(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0]; +static FUNC_PREFIX void pack_BN(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0]; + } } - } - } + } +} } -} // namespace internal_pack_BN namespace internal_pack_N { -static FUNC_PREFIX void -pack_N(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - double *RESTRICT _data_pdfs_20_31_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_20_31_11[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_N(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + double * RESTRICT _data_pdfs_20_31_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_20_31_11[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_N namespace internal_pack_TN { -static FUNC_PREFIX void -pack_TN(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0]; +static FUNC_PREFIX void pack_TN(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0]; + } } - } - } + } +} } -} // namespace internal_pack_TN namespace internal_pack_SE { -static FUNC_PREFIX void -pack_SE(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_SE(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_SE namespace internal_pack_BE { -static FUNC_PREFIX void -pack_BE(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_BE(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_BE namespace internal_pack_E { -static FUNC_PREFIX void -pack_E(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_E(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_E namespace internal_pack_TE { -static FUNC_PREFIX void -pack_TE(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_TE(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_TE namespace internal_pack_NE { -static FUNC_PREFIX void -pack_NE(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_NE(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_NE + + namespace internal_unpack_SW { -static FUNC_PREFIX void -unpack_SW(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_SW(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_SW namespace internal_unpack_BW { -static FUNC_PREFIX void -unpack_BW(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_BW(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_BW namespace internal_unpack_W { -static FUNC_PREFIX void -unpack_W(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_W(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_W namespace internal_unpack_TW { -static FUNC_PREFIX void -unpack_TW(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_TW(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_TW namespace internal_unpack_NW { -static FUNC_PREFIX void -unpack_NW(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_NW(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_NW namespace internal_unpack_BS { -static FUNC_PREFIX void -unpack_BS(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_BS(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_BS namespace internal_unpack_S { -static FUNC_PREFIX void -unpack_S(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - double *RESTRICT _data_pdfs_20_32_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_20_32_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_S(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + double * RESTRICT _data_pdfs_20_32_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_20_32_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_S namespace internal_unpack_TS { -static FUNC_PREFIX void -unpack_TS(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_TS(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_TS namespace internal_unpack_B { -static FUNC_PREFIX void -unpack_B(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - double *RESTRICT _data_pdfs_2m1_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_36; - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_2m1_36_10[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_B(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_36 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + double * RESTRICT _data_pdfs_2m1_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_36; + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_2m1_36_10[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_B namespace internal_unpack_T { -static FUNC_PREFIX void -unpack_T(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - double *RESTRICT _data_pdfs_21_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_35; - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_21_35_10[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_T(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + double * RESTRICT _data_pdfs_21_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_35; + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_21_35_10[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_T namespace internal_unpack_BN { -static FUNC_PREFIX void -unpack_BN(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_BN(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_BN namespace internal_unpack_N { -static FUNC_PREFIX void -unpack_N(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - double *RESTRICT _data_pdfs_20_31_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_20_31_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_N(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + double * RESTRICT _data_pdfs_20_31_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_20_31_11[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_N namespace internal_unpack_TN { -static FUNC_PREFIX void -unpack_TN(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_TN(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_TN namespace internal_unpack_SE { -static FUNC_PREFIX void -unpack_SE(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_SE(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_SE namespace internal_unpack_BE { -static FUNC_PREFIX void -unpack_BE(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_BE(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_BE namespace internal_unpack_E { -static FUNC_PREFIX void -unpack_E(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_E(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_E namespace internal_unpack_TE { -static FUNC_PREFIX void -unpack_TE(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_TE(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_TE namespace internal_unpack_NE { -static FUNC_PREFIX void -unpack_NE(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_NE(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } + } +} +} + + + +void PushPackInfo::pack(Direction dir, unsigned char * byte_buffer, IBlock * block) const +{ + double * buffer = reinterpret_cast(byte_buffer); + + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + + CellInterval ci; + pdfs->getSliceBeforeGhostLayer(dir, ci, 1, false); + + switch( dir ) + { + case stencil::SW: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_SW::pack_SW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BW: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_BW::pack_BW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::W: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_W::pack_W(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TW: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_TW::pack_TW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::NW: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_NW::pack_NW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BS: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_BS::pack_BS(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::S: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_S::pack_S(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TS: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_TS::pack_TS(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::B: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_B::pack_B(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::T: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_T::pack_T(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BN: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_BN::pack_BN(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::N: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_N::pack_N(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TN: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_TN::pack_TN(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::SE: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_SE::pack_SE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BE: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_BE::pack_BE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::E: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_E::pack_E(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TE: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_TE::pack_TE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::NE: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_NE::pack_NE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + + default: + WALBERLA_ASSERT(false); + } +} + + +void PushPackInfo::unpack(Direction dir, unsigned char * byte_buffer, IBlock * block) const +{ + double * buffer = reinterpret_cast(byte_buffer); + + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + + CellInterval ci; + pdfs->getGhostRegion(dir, ci, 1, false); + auto communciationDirection = stencil::inverseDir[dir]; + + switch( communciationDirection ) + { + case stencil::SW: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_SW::unpack_SW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BW: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_BW::unpack_BW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::W: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_W::unpack_W(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TW: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_TW::unpack_TW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::NW: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_NW::unpack_NW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BS: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_BS::unpack_BS(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::S: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_S::unpack_S(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TS: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_TS::unpack_TS(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::B: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_B::unpack_B(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::T: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_T::unpack_T(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BN: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_BN::unpack_BN(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::N: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_N::unpack_N(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TN: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_TN::unpack_TN(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::SE: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_SE::unpack_SE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BE: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_BE::unpack_BE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::E: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_E::unpack_E(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TE: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_TE::unpack_TE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::NE: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_NE::unpack_NE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + + default: + WALBERLA_ASSERT(false); } - } -} -} // namespace internal_unpack_NE - -void PushPackInfo::pack(Direction dir, unsigned char *byte_buffer, - IBlock *block) const { - double *buffer = reinterpret_cast(byte_buffer); - - auto pdfs = block->getData>(pdfsID); - - CellInterval ci; - pdfs->getSliceBeforeGhostLayer(dir, ci, 1, false); - - switch (dir) { - case stencil::SW: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_SW::pack_SW(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BW: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_BW::pack_BW(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::W: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_W::pack_W(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TW: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_TW::pack_TW(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::NW: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_NW::pack_NW(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BS: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_BS::pack_BS(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::S: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_S::pack_S(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TS: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_TS::pack_TS(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::B: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_B::pack_B(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::T: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_T::pack_T(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BN: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_BN::pack_BN(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::N: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_N::pack_N(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TN: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_TN::pack_TN(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::SE: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_SE::pack_SE(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BE: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_BE::pack_BE(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::E: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_E::pack_E(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TE: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_TE::pack_TE(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::NE: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_NE::pack_NE(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - default: - WALBERLA_ASSERT(false); - } -} - -void PushPackInfo::unpack(Direction dir, unsigned char *byte_buffer, - IBlock *block) const { - double *buffer = reinterpret_cast(byte_buffer); - - auto pdfs = block->getData>(pdfsID); - - CellInterval ci; - pdfs->getGhostRegion(dir, ci, 1, false); - auto communciationDirection = stencil::inverseDir[dir]; - - switch (communciationDirection) { - case stencil::SW: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_SW::unpack_SW( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BW: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_BW::unpack_BW( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::W: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_W::unpack_W(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TW: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_TW::unpack_TW( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::NW: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_NW::unpack_NW( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BS: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_BS::unpack_BS( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::S: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_S::unpack_S(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TS: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_TS::unpack_TS( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::B: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_B::unpack_B(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::T: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_T::unpack_T(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BN: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_BN::unpack_BN( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::N: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_N::unpack_N(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TN: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_TN::unpack_TN( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::SE: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_SE::unpack_SE( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BE: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_BE::unpack_BE( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::E: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_E::unpack_E(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TE: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_TE::unpack_TE( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::NE: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_NE::unpack_NE( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - default: - WALBERLA_ASSERT(false); - } -} - -uint_t PushPackInfo::size(stencil::Direction dir, const IBlock *block) const { - auto pdfs = block->getData>(pdfsID); - - CellInterval ci; - pdfs->getGhostRegion(dir, ci, 1, false); - - uint_t elementsPerCell = 0; - - switch (dir) { - case stencil::SW: - elementsPerCell = 1; - break; - - case stencil::BW: - elementsPerCell = 1; - break; - - case stencil::W: - elementsPerCell = 5; - break; - - case stencil::TW: - elementsPerCell = 1; - break; - - case stencil::NW: - elementsPerCell = 1; - break; - - case stencil::BS: - elementsPerCell = 1; - break; - - case stencil::S: - elementsPerCell = 5; - break; - - case stencil::TS: - elementsPerCell = 1; - break; - - case stencil::B: - elementsPerCell = 5; - break; - - case stencil::T: - elementsPerCell = 5; - break; - - case stencil::BN: - elementsPerCell = 1; - break; - - case stencil::N: - elementsPerCell = 5; - break; - - case stencil::TN: - elementsPerCell = 1; - break; - - case stencil::SE: - elementsPerCell = 1; - break; - - case stencil::BE: - elementsPerCell = 1; - break; - - case stencil::E: - elementsPerCell = 5; - break; - - case stencil::TE: - elementsPerCell = 1; - break; - - case stencil::NE: - elementsPerCell = 1; - break; - - default: - elementsPerCell = 0; - } - return ci.numCells() * elementsPerCell * sizeof(double); } + +uint_t PushPackInfo::size(stencil::Direction dir, const IBlock * block) const +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + + CellInterval ci; + pdfs->getGhostRegion(dir, ci, 1, false); + + uint_t elementsPerCell = 0; + + switch( dir ) + { + case stencil::SW: + elementsPerCell = 1; + break; + + case stencil::BW: + elementsPerCell = 1; + break; + + case stencil::W: + elementsPerCell = 5; + break; + + case stencil::TW: + elementsPerCell = 1; + break; + + case stencil::NW: + elementsPerCell = 1; + break; + + case stencil::BS: + elementsPerCell = 1; + break; + + case stencil::S: + elementsPerCell = 5; + break; + + case stencil::TS: + elementsPerCell = 1; + break; + + case stencil::B: + elementsPerCell = 5; + break; + + case stencil::T: + elementsPerCell = 5; + break; + + case stencil::BN: + elementsPerCell = 1; + break; + + case stencil::N: + elementsPerCell = 5; + break; + + case stencil::TN: + elementsPerCell = 1; + break; + + case stencil::SE: + elementsPerCell = 1; + break; + + case stencil::BE: + elementsPerCell = 1; + break; + + case stencil::E: + elementsPerCell = 5; + break; + + case stencil::TE: + elementsPerCell = 1; + break; + + case stencil::NE: + elementsPerCell = 1; + break; + + default: + elementsPerCell = 0; + } + return ci.numCells() * elementsPerCell * sizeof( double ); +} + + + } // namespace lbm } // namespace walberla \ No newline at end of file diff --git a/src/walberla_bridge/walberla_utils.hpp b/src/walberla_bridge/walberla_utils.hpp index fc55fa39a9a..f412d96fc67 100644 --- a/src/walberla_bridge/walberla_utils.hpp +++ b/src/walberla_bridge/walberla_utils.hpp @@ -25,6 +25,8 @@ #include #include +#include + #include #include diff --git a/testsuite/python/lb_shear.py b/testsuite/python/lb_shear.py index c1e73e43f8f..3ce39b27a83 100644 --- a/testsuite/python/lb_shear.py +++ b/testsuite/python/lb_shear.py @@ -17,7 +17,7 @@ import unittest as ut import unittest_decorators as utx import numpy as np -# import math WALBERLA TODO +import math import espressomd.lb import espressomd.lbboundaries @@ -144,9 +144,9 @@ def check_profile(self, shear_plane_normal, shear_direction): # speed of sound of the LB fluid in MD units (agrid/tau is due to # LB->MD unit conversion) -# speed_of_sound = 1. / math.sqrt(3.) * self.lbf.agrid / self.lbf.tau + speed_of_sound = 1. / math.sqrt(3.) * self.lbf.agrid / self.lbf.tau # equation of state for the LB fluid -# p_eq = speed_of_sound**2.0 * DENS + p_eq = speed_of_sound**2.0 * DENS # see Eq. 1.15 and 1.29 in # Krüger, Timm, et al. "The lattice Boltzmann method." Springer International Publishing 10 (2017): 978-3. # and @@ -157,16 +157,15 @@ def check_profile(self, shear_plane_normal, shear_direction): # '*' denotes the outer product and 'u' is the velocity field # NOTE: the so called stress property of the fluid is actually the # pressure tensor not the viscous stress tensor! -# shear_rate = SHEAR_VELOCITY / H + shear_rate = SHEAR_VELOCITY / H dynamic_viscosity = self.lbf.viscosity * DENS -# p_expected = p_eq * np.identity(3) - dynamic_viscosity * shear_rate * ( -# np.outer(shear_plane_normal, shear_direction) + np.transpose(np.outer(shear_plane_normal, shear_direction))) -# for n in (2, 3, 4), (3, 4, 2), (5, 4, 3): -# node_pressure_tensor = np.copy( -# self.lbf[n[0], n[1], n[2]].pressure_tensor) -# WALBERLA TODO -# np.testing.assert_allclose(node_pressure_tensor, -# p_expected, atol=1E-5, rtol=5E-3) + p_expected = p_eq * np.identity(3) - dynamic_viscosity * shear_rate * ( + np.outer(shear_plane_normal, shear_direction) + np.transpose(np.outer(shear_plane_normal, shear_direction))) + for n in (2, 3, 4), (3, 4, 2), (5, 4, 3): + node_pressure_tensor = np.copy( + self.lbf[n[0], n[1], n[2]].pressure_tensor) + np.testing.assert_allclose(node_pressure_tensor, + p_expected, atol=1E-5, rtol=5E-3) np.testing.assert_allclose( np.copy(wall1.get_force()),