From cf96b26c89a422ab75efa263759f017e0f9ac79b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 27 Apr 2026 10:21:38 -0700 Subject: [PATCH] Fix GPU dispatch for PolymorphicArenaAllocator in DefaultInitialization Follow-up from transitioning to polymorphic particle containers. `amrex::RunOnGpu` is `false_type` because AMReX does not specialize `RunOnGpu` for this allocator. This causes `DefaultInitializeRuntimeAttributes` to take the CPU code path even when particle data lives on device, leading to segfaults when ionization or user-defined runtime attributes are used with SYCL (and potentially other GPU backends that don't have UVA). This affected users that used runtime attributes on hardware that did not successfully fell back to unified memory (and we try to avoid that anyway). Co-Authored-By: Tim Williams --- .../ParticleCreation/DefaultInitialization.H | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Source/Particles/ParticleCreation/DefaultInitialization.H b/Source/Particles/ParticleCreation/DefaultInitialization.H index 76b40cf5252..dcefaae1d61 100644 --- a/Source/Particles/ParticleCreation/DefaultInitialization.H +++ b/Source/Particles/ParticleCreation/DefaultInitialization.H @@ -144,6 +144,11 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, constexpr int lev = 0; const amrex::Real t = WarpX::GetInstance().gett_new(lev); + // Check: does the particle data reside on CPU or GPU? + amrex::Arena const * const pc_arena = pc.arena(); + const bool run_on_gpu = + pc_arena->isManaged() || pc_arena->isDevice(); + // Initialize the last NumRuntimeRealComps() - n_external_attr_real runtime real attributes for (int j = PIdx::nattribs + n_external_attr_real; j < ptile.NumRealComps() ; ++j) { @@ -158,7 +163,7 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, const QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt = p_qs_engine->build_optical_depth_functor(); // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel - if constexpr (amrex::RunOnGpu>::value) { + if (run_on_gpu) { amrex::ParallelForRNG(stop - start, [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept { const int ip = i + start; @@ -185,7 +190,7 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, const BreitWheelerGetOpticalDepth breit_wheeler_get_opt = p_bw_engine->build_optical_depth_functor();; // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel - if constexpr (amrex::RunOnGpu>::value) { + if (run_on_gpu) { amrex::ParallelForRNG(stop - start, [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept { const int ip = i + start; @@ -214,7 +219,7 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, const amrex::ParserExecutor<7> user_real_attrib_parserexec = user_real_attrib_parser[ia]->compile<7>(); // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel - if constexpr (amrex::RunOnGpu>::value) { + if (run_on_gpu) { amrex::ParallelFor(stop - start, [=] AMREX_GPU_DEVICE (int i) noexcept { const int ip = i + start; @@ -246,7 +251,7 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, if (it_ioniz != particle_icomps.end() && std::distance(particle_icomps.begin(), it_ioniz) == j) { - if constexpr (amrex::RunOnGpu>::value) { + if (run_on_gpu) { amrex::ParallelFor(stop - start, [=] AMREX_GPU_DEVICE (int i) noexcept { const int ip = i + start; @@ -268,7 +273,7 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, { const amrex::ParserExecutor<7> user_int_attrib_parserexec = user_int_attrib_parser[ia]->compile<7>(); - if constexpr (amrex::RunOnGpu>::value) { + if (run_on_gpu) { amrex::ParallelFor(stop - start, [=] AMREX_GPU_DEVICE (int i) noexcept { const int ip = i + start;