diff --git a/dependencies.py b/dependencies.py index 114cfd145..d41bb9567 100755 --- a/dependencies.py +++ b/dependencies.py @@ -367,8 +367,10 @@ def on_install_confirmed(settings: Settings) -> None: "kokkos", settings.kokkos_version, settings.kokkos_backend, - settings.kokkos_arch.strip().lower(), ) + if settings.kokkos_arch.strip().lower(): + kokkos_modfile_file = os.path.join(kokkos_modfile_file, settings.kokkos_arch.strip().lower()) + os.makedirs(os.path.dirname(kokkos_modfile_file), exist_ok=True) if os.path.exists(kokkos_modfile_file) and not settings.overwrite: raise FileExistsError( diff --git a/input.example.toml b/input.example.toml index 8d6f25117..841671a35 100644 --- a/input.example.toml +++ b/input.example.toml @@ -7,7 +7,7 @@ # Simulation engine to use # @required # @type: string - # @enum: "SRPIC", "GRPIC" + # @enum: "SRPIC", "GRPIC", "HYBRID" engine = "" # Max runtime in physical (code) units # @required @@ -224,6 +224,78 @@ # @from: `scales.larmor0` # @value: `1 / larmor0` +# Electron-fluid closure for the hybrid Ohm's law +# @note: Only read by the `HYBRID` engine (ignored by `SRPIC`/`GRPIC`) +[hybrid] + # Adiabatic index of the massless electron fluid + # @type: float [> 0.0] + # @default: 1.66667 (= 5/3) + # @note: `1.0` corresponds to an isothermal closure + gamma_ad = "" + # Electron temperature `T_e` in physical (code) units + # @type: float [>= 0.0] + # @default: 0.0 + # @note: `0.0` corresponds to cold electrons + theta0 = "" + # Vacuum threshold for the hybrid Ohm's law, in units of `n0` + # @type: float [> 0.0] + # @default: 1e-3 + # @note: Above it `E` follows the usual `[...]/N`; below it `E` ramps continuously + # to zero so plasma-free cells keep `B` frozen instead of amplifying the + # right-hand side by `1/dens_min` (statistically-empty or unfilled ghost + # cells, a fraction `e^-ppc` for finite `ppc`) + dens_min = "" + # Cap on the local whistler Courant the Ohm's-law Hall term may imply + # @type: float + # @default: 0.5 + # @note: The implied whistler speed `v_w ~ |B|/N` can outrun `dx/dt` at shock + # overshoots and magnetic cavities; the Hall term is scaled so its + # whistler Courant never exceeds this. Identity in resolvable cells + # @note: `<= 0.0` disables the limiter + hall_lim = "" + # Vacuum resistivity, in physical (code) units + # @type: float [>= 0.0] + # @default: 0.0 + # @note: `> 0.0` makes the sub-`dens_min` vacuum resistive: `E` crossfades to + # `eta * curl B` below the threshold, so vacuum regions diffuse toward a + # curl-free field instead of freezing. Needed when fast plasma sweeps + # along a vacuum boundary, where a hard `E = 0` mask pumps `B` at `~ u/dx` + # @note: Internally clamped to the explicit-diffusion stability limit; any value is safe + # @note: `0.0` disables it (hard `E = 0` vacuum) + resist_vac = "" + # Hyper-resistivity coefficient `eta_H` (adds `-eta_H grad^4 B` to the field advance) + # @type: float [>= 0.0] + # @default: 0.0 + # @note: Damps the few-cell band the moments filter leaves without kinetic + # feedback, while leaving physical scales (`k dx < 0.3`) essentially + # untouched; size so `eta_H (pi/dx)^4` exceeds the drive + # @note: Internally clamped to the explicit `grad^4` stability limit; any value is safe + # @note: `0.0` disables it + resist_hyper = "" + # Pegasus-style sub-cycled magnetic-field advance + # @type: bool + # @default: true + # @note: Integrates the whistler-stiff Ohm/Faraday loop with adaptive SSP-RK3 + # sub-steps at its own CFL; `false` uses the legacy 3-push scheme + subcycle = "" + # Target whistler Courant per field sub-step + # @type: float [> 0.0] + # @default: 0.5 + # @note: Only read when `subcycle = true` + subcycle_courant = "" + # Cap on the number of field sub-steps per advance + # @type: int [>= 1] + # @default: 64 + # @note: Beyond the cap the per-cell Hall limiter (`hall_lim`) takes over + subcycle_max = "" + # Characteristic flow speed for the hybrid CFL, in physical (code) units + # @type: float [>= 0.0] + # @default: 0.0 + # @note: Added to the Alfven + whistler signal speeds in the Pegasus CFL + # `dt <= CFL * min(dx0 / v_max, 2*pi/Omega)` [Kunz, Stone & Bai (2014), ยง3.5] + # @note: `0.0` lets the Alfven + whistler speeds set `dt`; raise it for super-whistler flows + v_max = "" + [radiation] [radiation.drag] [radiation.drag.synchrotron] diff --git a/pgens/hybrid/hybrid.toml b/pgens/hybrid/hybrid.toml new file mode 100644 index 000000000..e670e3a8e --- /dev/null +++ b/pgens/hybrid/hybrid.toml @@ -0,0 +1,50 @@ +[simulation] + name = "hybrid" + engine = "hybrid" + runtime = 1.0 + +[grid] + resolution = [512, 256] + extent = [[0.0, 2.0], [0.0, 1.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["PERIODIC"], ["PERIODIC"]] + particles = [["PERIODIC"], ["PERIODIC"]] + +[scales] + larmor0 = 1.0 + skindepth0 = 1.0 + +[algorithms] + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 10.0 + + [[particles.species]] + label = "ions" + mass = 1.0 + charge = 1.0 + maxnpart = 1e5 + +[setup] + +[output] + interval_time = 0.1 + + [output.fields] + quantities = ["N_1", "B"] + + [output.particles] + species = [1] + stride = 10 + +[checkpoint] + keep = 0 + +[diagnostics] + colored_stdout = true diff --git a/pgens/hybrid/pgen.hpp b/pgens/hybrid/pgen.hpp new file mode 100644 index 000000000..e3359fefa --- /dev/null +++ b/pgens/hybrid/pgen.hpp @@ -0,0 +1,30 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "traits/pgen.h" + +#include "framework/domain/metadomain.h" +#include "framework/parameters/parameters.h" + +namespace user { + + template + struct PGen { + static constexpr auto D { M::Dim }; + + static constexpr auto engines = + ::traits::pgen::compatible_with {}; + static constexpr auto metrics = + ::traits::pgen::compatible_with {}; + static constexpr auto dimensions = + ::traits::pgen::compatible_with {}; + + PGen(const SimulationParams&, const Metadomain&) {} + }; + +} // namespace user + +#endif diff --git a/pgens/shock/pgen.hpp b/pgens/shock/pgen.hpp index 7a7aaac21..4553b7d37 100644 --- a/pgens/shock/pgen.hpp +++ b/pgens/shock/pgen.hpp @@ -259,7 +259,7 @@ namespace user { domain.fields.em, init_flds, domain.mesh.metric }); - metadomain.CommunicateFields(domain, Comm::E | Comm::B); + metadomain.CommunicateFields(domain, Comm::EM); /* tag particles inside the injection zone as dead diff --git a/pgens/streaming/pgen.hpp b/pgens/streaming/pgen.hpp index 6983d249d..470cc780f 100644 --- a/pgens/streaming/pgen.hpp +++ b/pgens/streaming/pgen.hpp @@ -54,7 +54,8 @@ namespace user { struct PGen { static constexpr auto D { M::Dim }; // compatibility traits for the problem generator - static constexpr auto engines = ::traits::pgen::compatible_with {}; + static constexpr auto engines = + ::traits::pgen::compatible_with {}; static constexpr auto metrics = ::traits::pgen::compatible_with {}; static constexpr auto dimensions = @@ -87,13 +88,13 @@ namespace user { raise::ErrorIf(nspec % 2 != 0, "Number of species must be even for this setup", HERE); - for (auto n = 0u; n < nspec; n += 2) { - raise::ErrorIf( - global_domain.species_params()[n].charge() != - -global_domain.species_params()[n + 1].charge(), - "Charges of i-th and i+1-th species must be opposite for this setup", - HERE); - } + // for (auto n = 0u; n < nspec; n += 2) { + // raise::ErrorIf( + // global_domain.species_params()[n].charge() != + // -global_domain.species_params()[n + 1].charge(), + // "Charges of i-th and i+1-th species must be opposite for this + // setup", HERE); + // } for (auto* specs : { &drifts_in_x, &drifts_in_y, &drifts_in_z, &temperatures }) { if (specs->empty()) { @@ -126,13 +127,29 @@ namespace user { const auto drift_2 = prmvec_t { drifts_in_x[n + 1], drifts_in_y[n + 1], drifts_in_z[n + 1] }; - arch::InjectUniformMaxwellians( + const auto edist1 = arch::energy_dist::MaxwellianNonRel { + domain.random_pool(), + temperatures[n], + drift_1 + }; + const auto edist2 = arch::energy_dist::MaxwellianNonRel { + domain.random_pool(), + temperatures[n + 1], + drift_2 + }; + arch::InjectUniform( params, domain, - densities[n / 2], - { temperatures[n], temperatures[n + 1] }, - { n + 1, n + 2 }, - { drift_1, drift_2 }); + { 1u, 2u }, + { edist1, edist2 }, + ONE); + // arch::InjectUniformMaxwellians( + // params, + // domain, + // densities[n / 2], + // { temperatures[n], temperatures[n + 1] }, + // { n + 1, n + 2 }, + // { drift_1, drift_2 }); } } }; diff --git a/src/archetypes/energy_dist.h b/src/archetypes/energy_dist.h index 11fe04369..7af8dfd03 100644 --- a/src/archetypes/energy_dist.h +++ b/src/archetypes/energy_dist.h @@ -76,6 +76,38 @@ namespace arch::energy_dist { random_number_pool_t pool; }; + Inline void NonRelMaxwellian(vec_t& v, + real_t temp, + const random_number_pool_t& pool) { + auto rand_gen = pool.get_state(); + // Juttner-Synge distribution using the Box-Muller method - non-relativistic + real_t randX1 = Random(rand_gen); + while (cmp::AlmostZero(randX1)) { + randX1 = Random(rand_gen); + } + randX1 = math::sqrt(-TWO * math::log(randX1)); + real_t randX2 = static_cast(constant::TWO_PI) * + Random(rand_gen); + v[0] = randX1 * math::cos(randX2) * math::sqrt(temp); + + randX1 = Random(rand_gen); + while (cmp::AlmostZero(randX1)) { + randX1 = Random(rand_gen); + } + randX1 = math::sqrt(-TWO * math::log(randX1)); + randX2 = static_cast(constant::TWO_PI) * Random(rand_gen); + v[1] = randX1 * math::cos(randX2) * math::sqrt(temp); + + randX1 = Random(rand_gen); + while (cmp::AlmostZero(randX1)) { + randX1 = Random(rand_gen); + } + randX1 = math::sqrt(-TWO * math::log(randX1)); + randX2 = static_cast(constant::TWO_PI) * Random(rand_gen); + v[2] = randX1 * math::cos(randX2) * math::sqrt(temp); + pool.free_state(rand_gen); + } + Inline void JuttnerSinge(vec_t& v, real_t temp, const random_number_pool_t& pool) { @@ -291,6 +323,48 @@ namespace arch::energy_dist { short drift_dir { 0 }; }; + template + struct MaxwellianNonRel { + MaxwellianNonRel(random_number_pool_t& pool, + real_t temperature, + const std::vector& drift_vel = { ZERO, ZERO, ZERO }) + : pool { pool } + , temperature { temperature } { + raise::ErrorIf(drift_vel.size() != 3, + "Maxwellian: Drift velocity must be a 3D vector", + HERE); + raise::ErrorIf(temperature < ZERO, + "Maxwellian: Temperature must be non-negative", + HERE); + if (not cmp::AlmostZero_host(NORM(drift_vel[0], drift_vel[1], drift_vel[2]))) { + drift_vel_x1 = drift_vel[0]; + drift_vel_x2 = drift_vel[1]; + drift_vel_x3 = drift_vel[2]; + } + } + + Inline void operator()(const coord_t&, vec_t& v) const { + + if (cmp::AlmostZero(temperature)) { + v[0] = ZERO; + v[1] = ZERO; + v[2] = ZERO; + } else { + NonRelMaxwellian(v, temperature, pool); + } + v[0] += drift_vel_x1; + v[1] += drift_vel_x2; + v[2] += drift_vel_x3; + } + + private: + random_number_pool_t pool; + + const real_t temperature; + + real_t drift_vel_x1 { ZERO }, drift_vel_x2 { ZERO }, drift_vel_x3 { ZERO }; + }; + } // namespace arch::energy_dist #endif // ARCHETYPES_ENERGY_DIST_HPP diff --git a/src/archetypes/field_setter.h b/src/archetypes/field_setter.h index 1e3137d01..0758c3221 100644 --- a/src/archetypes/field_setter.h +++ b/src/archetypes/field_setter.h @@ -27,6 +27,7 @@ #include "arch/kokkos_aliases.h" #include "traits/archetypes.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/numeric.h" @@ -64,7 +65,7 @@ namespace arch { if constexpr (D == Dim::_1D) { const auto i1_ = COORD(i1); coord_t x_Phys { ZERO }; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { if constexpr (HasEx1) { metric.template convert({ i1_ + HALF }, x_Phys); EM(i1, em::ex1) = metric.template transform<1, Idx::T, Idx::U>( @@ -114,7 +115,7 @@ namespace arch { const auto i1_ = COORD(i1); const auto i2_ = COORD(i2); // srpic - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { coord_t x_Phys { ZERO }; if constexpr (HasEx1) { metric.template convert({ i1_ + HALF, i2_ }, x_Phys); @@ -153,7 +154,7 @@ namespace arch { { i1_ + HALF, i2_ + HALF }, finit.bx3(x_Phys)); } - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::UserFieldsInContravariantBasis) { // grpic if constexpr (HasDx1 && HasDx2 && HasDx3) { const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; @@ -203,7 +204,7 @@ namespace arch { const auto i2_ = COORD(i2); const auto i3_ = COORD(i3); coord_t x_Phys { ZERO }; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { // srpic if constexpr (HasEx1) { metric.template convert({ i1_ + HALF, i2_, i3_ }, @@ -250,7 +251,7 @@ namespace arch { { i1_ + HALF, i2_ + HALF, i3_ }, finit.bx3(x_Phys)); } - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::UserFieldsInContravariantBasis) { // grpic const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( @@ -295,6 +296,9 @@ namespace arch { template FS> struct CustomSetEMFields_kernel { + static_assert(::traits::engine::UserFieldsInTetradBasis, + "CustomSetEMFields_kernel is only implemented for SimEngines " + "with user-defined fields in tetrad basis"); static constexpr auto D = M::Dim; static constexpr auto HasEx1 = ::traits::fieldsetter::HasConditionalEx1; static constexpr auto HasEx2 = ::traits::fieldsetter::HasConditionalEx2; diff --git a/src/archetypes/moving_window.h b/src/archetypes/moving_window.h index dfbc57131..1e31d664f 100644 --- a/src/archetypes/moving_window.h +++ b/src/archetypes/moving_window.h @@ -267,7 +267,7 @@ namespace arch { } // synch ghost zones after moving the window - metadomain.CommunicateFields(domain, Comm::E | Comm::B); + metadomain.CommunicateFields(domain, Comm::EM); // communicate particles after moving metadomain.CommunicateParticles(domain); diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 415da851e..f6d21369e 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -262,6 +262,75 @@ namespace arch { } } + /** + * @brief Injects uniform number density of a single species everywhere in the domain + * @note for engines with a lone species (e.g. HYBRID ions), where the + * charge-neutral pair injector does not apply + * @param params Simulation parameters + * @param domain Domain object + * @param species Species index (1-based) + * @param energy_dist Energy distribution object + * @param number_density Number density (in units of n0) + * @param use_weights Use weights + * @param box Region to inject the particles in global coords + * @tparam S Simulation engine type + * @tparam M Metric type + * @tparam ED Energy distribution type + */ + template ED> + inline void InjectUniform(const SimulationParams& params, + Domain& domain, + spidx_t species, + const ED& energy_dist, + real_t number_density, + bool use_weights = false, + const boundaries_t& box = {}) { + raise::ErrorIf((M::CoordType != Coord::Cartesian) && (not use_weights), + "Weights must be used for non-Cartesian coordinates", + HERE); + raise::ErrorIf((M::CoordType == Coord::Cartesian) && use_weights, + "Weights should not be used for Cartesian coordinates", + HERE); + raise::ErrorIf(params.template get("particles.use_weights") != use_weights, + "Weights must be enabled from the input file to use them in " + "the injector", + HERE); + + { + boundaries_t nonempty_box; + for (auto d { 0u }; d < M::Dim; ++d) { + if (d < box.size()) { + nonempty_box.emplace_back(box[d].first, box[d].second); + } else { + nonempty_box.push_back(Range::All); + } + } + const auto result = ComputeNumInject(params, domain, number_density, nonempty_box); + if (not std::get<0>(result)) { + return; + } + const auto nparticles = std::get<1>(result); + const auto xi_min = std::get<2>(result); + const auto xi_max = std::get<3>(result); + + Kokkos::parallel_for("InjectUniform", + nparticles, + kernel::UniformInjectorSingle_kernel( + domain.species[species - 1], + domain.index(), + domain.mesh.metric, + xi_min, + xi_max, + energy_dist, + ONE / params.template get("scales.V0"), + domain.random_pool())); + domain.species[species - 1].set_npart( + domain.species[species - 1].npart() + nparticles); + domain.species[species - 1].set_counter( + domain.species[species - 1].counter() + nparticles); + } + } + /** * @brief Injects particles from a globally-defined map * @note very inefficient, should only be used for debug purposes diff --git a/src/archetypes/utils.h b/src/archetypes/utils.h index 1ae775309..7013d52e0 100644 --- a/src/archetypes/utils.h +++ b/src/archetypes/utils.h @@ -170,7 +170,7 @@ namespace arch { template inline void UpdateEMFields(Domain& domain, const F& fieldsetter) { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { Kokkos::deep_copy(domain.fields.bckp, domain.fields.em); Kokkos::parallel_for( "UpdateEMFields", @@ -181,7 +181,9 @@ namespace arch { fieldsetter)); // comm here } else { - raise::Error("Custom fieldsetter is only implemented for SRPIC", HERE); + raise::Error("Custom fieldsetter is only implemented for simengine which " + "defines user fields in tetrad basis", + HERE); } } diff --git a/src/engines/grpic/currents.h b/src/engines/grpic/currents.h index cb4032f54..fb02babea 100644 --- a/src/engines/grpic/currents.h +++ b/src/engines/grpic/currents.h @@ -264,7 +264,7 @@ namespace ntt { domain.fields.buff, size, domain.mesh.flds_bc())); - metadomain.CommunicateFields(domain, Comm::J); // J0 + metadomain.CommunicateFields(domain, ::Comm::CUR0); // J0 } } diff --git a/src/engines/grpic/grpic.hpp b/src/engines/grpic/grpic.hpp index 3487d2867..4f2bdd82a 100644 --- a/src/engines/grpic/grpic.hpp +++ b/src/engines/grpic/grpic.hpp @@ -38,6 +38,16 @@ namespace ntt { + namespace grpic::Comm { + static constexpr auto E = ::Comm::AUX_012; + static constexpr auto H = ::Comm::AUX_345; + static constexpr auto D = ::Comm::EM_012; + static constexpr auto D0 = ::Comm::EM0_012; + static constexpr auto B = ::Comm::EM_345; + static constexpr auto B0 = ::Comm::EM0_345; + static constexpr auto J = ::Comm::CUR0; + } // namespace grpic::Comm + template class GRPICEngine : public Engine { using base_t = Engine; @@ -64,6 +74,7 @@ namespace ntt { ~GRPICEngine() override = default; void step_forward(timer::Timers& timers, domain_t& dom) override { + const auto fieldsolver_enabled = m_params.template get( "algorithms.fieldsolver.enable"); const auto deposit_enabled = m_params.template get( @@ -94,7 +105,8 @@ namespace ntt { * em0::D, em::D, em0::B, em::B <- boundary conditions */ m_metadomain.CommunicateFields(dom, - Comm::B | Comm::B0 | Comm::D | Comm::D0); + grpic::Comm::B | grpic::Comm::B0 | + grpic::Comm::D | grpic::Comm::D0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -122,7 +134,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::H | Comm::E); + m_metadomain.CommunicateFields(dom, grpic::Comm::H | grpic::Comm::E); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -144,7 +156,7 @@ namespace ntt { /** * em0::B, em::B <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + m_metadomain.CommunicateFields(dom, grpic::Comm::B | grpic::Comm::B0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -166,7 +178,7 @@ namespace ntt { /** * em0::D, em::D <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, grpic::Comm::D | grpic::Comm::D0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -186,7 +198,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::H | Comm::E); + m_metadomain.CommunicateFields(dom, grpic::Comm::H | grpic::Comm::E); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -209,7 +221,7 @@ namespace ntt { /** * em0::B, em::B <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + m_metadomain.CommunicateFields(dom, grpic::Comm::B | grpic::Comm::B0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -226,7 +238,7 @@ namespace ntt { /** * em0::D, em::D <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, grpic::Comm::D | grpic::Comm::D0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -243,7 +255,7 @@ namespace ntt { /** * aux::H <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::H); + m_metadomain.CommunicateFields(dom, grpic::Comm::H); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -261,7 +273,7 @@ namespace ntt { /** * em0::D, em::D <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, grpic::Comm::D | grpic::Comm::D0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -336,7 +348,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::E); + m_metadomain.CommunicateFields(dom, grpic::Comm::E); timers.stop("Communications"); timers.start("FieldBoundaries"); /** @@ -360,7 +372,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + m_metadomain.CommunicateFields(dom, grpic::Comm::B | grpic::Comm::B0); timers.stop("Communications"); /** * em0::B, em::B <- boundary conditions @@ -384,7 +396,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::H); + m_metadomain.CommunicateFields(dom, grpic::Comm::H); timers.stop("Communications"); timers.start("FieldBoundaries"); /** @@ -422,8 +434,8 @@ namespace ntt { timers.stop("CurrentDeposit"); timers.start("Communications"); - m_metadomain.SynchronizeFields(dom, Comm::J); - m_metadomain.CommunicateFields(dom, Comm::J); + m_metadomain.SynchronizeFields(dom, grpic::Comm::J); + m_metadomain.CommunicateFields(dom, grpic::Comm::J); timers.stop("Communications"); timers.start("FieldBoundaries"); @@ -464,7 +476,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::E); + m_metadomain.CommunicateFields(dom, grpic::Comm::E); timers.stop("Communications"); timers.start("FieldBoundaries"); /** @@ -492,7 +504,7 @@ namespace ntt { * em0::B, em::B <- boundary conditions */ timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + m_metadomain.CommunicateFields(dom, grpic::Comm::B | grpic::Comm::B0); timers.stop("Communications"); timers.start("FieldBoundaries"); grpic::FieldBoundaries(dom, @@ -530,7 +542,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, grpic::Comm::D | grpic::Comm::D0); timers.stop("Communications"); timers.start("FieldBoundaries"); grpic::FieldBoundaries(dom, @@ -551,7 +563,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::H); + m_metadomain.CommunicateFields(dom, grpic::Comm::H); timers.stop("Communications"); timers.start("FieldBoundaries"); /** @@ -601,7 +613,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, grpic::Comm::D | grpic::Comm::D0); timers.stop("Communications"); timers.start("FieldBoundaries"); grpic::FieldBoundaries(dom, diff --git a/src/engines/hybrid/fields_bcs.h b/src/engines/hybrid/fields_bcs.h new file mode 100644 index 000000000..f9a4d6ec4 --- /dev/null +++ b/src/engines/hybrid/fields_bcs.h @@ -0,0 +1,368 @@ +/** + * @file engines/hybrid/fields_bcs.h + * @brief Non-periodic (reflecting-wall / perfect-conductor) boundary + * conditions for the HYBRID engine, x1 walls only. + * @implements + * - ntt::hybrid::PerfectConductorFieldsIn<> -> void + * - ntt::hybrid::FieldBoundaries<> -> void (em: Ee and/or Bf) + * - ntt::hybrid::WallEPrime<> -> void (em0::345 = Ee'/Ee'') + * - ntt::hybrid::WallScratchB<> -> void (cur = Bf*, Bf**) + * - ntt::hybrid::WallBckpFill<> -> void (bckp = Ec'/Bc') + * - ntt::hybrid::MomentsWallBC<> -> void (aux = V, N) + * @namespaces: + * - ntt::hybrid:: + * + * PERIODIC boundaries are realized by the metadomain halo exchange + * (Metadomain::CommunicateFields) and are skipped here. + * + * The conductor wall (kernels/hybrid/wall_bcs.hpp) enforces E_tan = 0 on the + * wall plane; Faraday then freezes the wall-plane B_n automatically, which is + * the correct condition for an OBLIQUE background field. The wall-plane B_n is + * deliberately never overwritten (zeroing it is only valid for perpendicular + * fields and plants a div(B) layer at the wall). B ghosts get an even + * (zero-gradient) mirror so the Hall term sees no fake wall current. + * + * EVERY buffer whose wall ghosts (or wall-plane values) enter a stencil needs + * explicit treatment -- there is no halo exchange at a physical wall, so + * untreated ghosts keep their allocation value (zero) forever: + * - em (Ee/Bf) -> FieldBoundaries (BC::E and/or BC::B) + * - em0::345 (Ee'/Ee'') -> WallEPrime, before Faraday #2/#3 + * - cur (Bf*, Bf**) -> WallScratchB, before EMF #1/#2 (Hall stencil) + * - bckp (Ec'/Bc') -> WallBckpFill, before the particle gathers + * - aux (V/N) -> MomentsWallBC after each deposit: folds the + * ghost deposit tails back as the image-plasma contribution (V_x odd) and + * mirror-fills the ghosts for the EMF / filter stencils + * + * @note Hybrid is Cartesian Minkowski, so only the conductor wall is + * meaningful (no axis / GR-horizon boundaries). + * @note Only x1 walls are implemented (the hybrid shock geometry); a + * CONDUCTOR boundary in x2/x3 raises. + */ + +#ifndef ENGINES_HYBRID_FIELDS_BCS_H +#define ENGINES_HYBRID_FIELDS_BCS_H + +#include "enums.h" +#include "global.h" + +#include "arch/directions.h" +#include "arch/kokkos_aliases.h" +#include "utils/error.h" + +#include "metrics/minkowski.h" + +#include "framework/domain/domain.h" +#include "framework/domain/grid.h" +#include "kernels/hybrid/wall_bcs.hpp" + +#include + +#include + +namespace ntt { + namespace hybrid { + + namespace wall { + + /** + * @brief Range over the wall layer for x1-staggered (edge-E / face-B) + * kernels: i1 = 0 (wall plane) .. N_GHOSTS (ghost layers; the + * +x1 side has one node less), full transverse extent. + */ + template + auto StaggeredRange(Domain>& domain, + short sign) -> range_t { + std::vector xi_min, xi_max; + xi_min.push_back(0); + xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); + const std::vector all_dirs { in::x1, in::x2, in::x3 }; + for (auto d { 1u }; d < static_cast(D); ++d) { + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(all_dirs[d])); + } + if constexpr (D == Dim::_1D) { + return CreateRangePolicy({ xi_min[0] }, { xi_max[0] }); + } else if constexpr (D == Dim::_2D) { + return CreateRangePolicy({ xi_min[0], xi_min[1] }, + { xi_max[0], xi_max[1] }); + } else { + return CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + { xi_max[0], xi_max[1], xi_max[2] }); + } + } + + /** + * @brief Range over the wall layer for cell-centered kernels: + * i1 = ghost layer 0 .. N_GHOSTS; transverse extent is + * active-only when `active_transverse` (the moment fold; corner + * tails are already remapped by the transverse sync) and full + * otherwise. + */ + template + auto CellRange(Domain>& domain, + bool active_transverse) -> range_t { + std::vector xi_min, xi_max; + xi_min.push_back(0); + xi_max.push_back(N_GHOSTS); + const std::vector all_dirs { in::x1, in::x2, in::x3 }; + for (auto d { 1u }; d < static_cast(D); ++d) { + const auto dd = all_dirs[d]; + if (active_transverse) { + xi_min.push_back(domain.mesh.i_min(dd)); + xi_max.push_back(domain.mesh.i_max(dd)); + } else { + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); + } + } + if constexpr (D == Dim::_1D) { + return CreateRangePolicy({ xi_min[0] }, { xi_max[0] }); + } else if constexpr (D == Dim::_2D) { + return CreateRangePolicy({ xi_min[0], xi_min[1] }, + { xi_max[0], xi_max[1] }); + } else { + return CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + { xi_max[0], xi_max[1], xi_max[2] }); + } + } + + /** + * @brief Invoke `f(sign, i_edge)` for every x1 conductor wall this rank + * owns. Raises on non-x1 conductor walls (unimplemented). + */ + template + void ForEachX1Wall(Domain>& domain, + const Grid& global_grid, + F f) { + for (const auto& direction : dir::Directions::orth) { + const auto gbc = global_grid.flds_bc_in(direction); + if (gbc == FldsBC::PERIODIC) { + continue; // realized by the halo exchange + } else if (gbc == FldsBC::CONDUCTOR) { + if (domain.mesh.flds_bc_in(direction) != FldsBC::CONDUCTOR) { + continue; // not an edge-owning rank + } + const auto dim = direction.get_dim(); + const auto sign = direction.get_sign(); + raise::ErrorIf(dim != in::x1, + "hybrid engine: CONDUCTOR walls are only " + "implemented in x1", + HERE); + const ncells_t i_edge = (sign > 0) ? domain.mesh.i_max(in::x1) + : domain.mesh.i_min(in::x1); + f(sign, i_edge); + } else { + raise::Error("hybrid engine: only PERIODIC and CONDUCTOR field " + "boundaries are implemented", + HERE); + } + } + } + + } // namespace wall + + /** + * @brief Apply the perfect-conductor wall condition on `em` along one + * orthogonal direction: BC::E -> Ee (comps 0..2): E_tan = 0 on the + * wall plane + mirror ghosts; BC::B -> Bf (comps 3..5): even mirror + * ghosts (the wall-plane B_n is left to Faraday, which freezes it + * because of E_tan = 0). + */ + template + void PerfectConductorFieldsIn( + const dir::direction_t& direction, + Domain>& domain, + BCTags tags) { + const auto sign = direction.get_sign(); + raise::ErrorIf(direction.get_dim() != in::x1, + "hybrid engine: CONDUCTOR walls are only implemented in x1", + HERE); + const ncells_t i_edge = (sign > 0) ? domain.mesh.i_max(in::x1) + : domain.mesh.i_min(in::x1); + const auto range = wall::StaggeredRange(domain, sign); + if (tags & BC::E) { + if (sign > 0) { + Kokkos::parallel_for( + "WallEdgeE", + range, + kernel::hybrid::WallEdgeE_kernel(domain.fields.em, i_edge, 0)); + } else { + Kokkos::parallel_for( + "WallEdgeE", + range, + kernel::hybrid::WallEdgeE_kernel(domain.fields.em, i_edge, 0)); + } + } + if (tags & BC::B) { + if (sign > 0) { + Kokkos::parallel_for( + "WallFaceB", + range, + kernel::hybrid::WallFaceB_kernel(domain.fields.em, + i_edge, + 3)); + } else { + Kokkos::parallel_for( + "WallFaceB", + range, + kernel::hybrid::WallFaceB_kernel(domain.fields.em, + i_edge, + 3)); + } + } + } + + /** + * @brief Apply non-periodic field boundaries to the evolved `em` fields. + * @param global_grid the global grid (for the configured boundary type) + * @param tags which fields to act on (BC::E, BC::B, or both) + */ + template + void FieldBoundaries(Domain>& domain, + const Grid& global_grid, + BCTags tags) { + for (const auto& direction : dir::Directions::orth) { + const auto gbc = global_grid.flds_bc_in(direction); + if (gbc == FldsBC::PERIODIC) { + continue; // realized by the halo exchange + } else if (gbc == FldsBC::CONDUCTOR) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CONDUCTOR) { + PerfectConductorFieldsIn(direction, domain, tags); + } + } else { + raise::Error("hybrid engine: only PERIODIC and CONDUCTOR field " + "boundaries are implemented", + HERE); + } + } + } + + /** + * @brief Conductor wall condition on the half-step edge-E (em0::345 = + * Ee'/Ee''). Call after EMF #1/#2, before the Faraday push that + * consumes it. + */ + template + void WallEPrime(Domain>& domain, + const Grid& global_grid) { + wall::ForEachX1Wall(domain, global_grid, [&](short sign, ncells_t i_edge) { + const auto range = wall::StaggeredRange(domain, sign); + if (sign > 0) { + Kokkos::parallel_for( + "WallEdgeE", + range, + kernel::hybrid::WallEdgeE_kernel(domain.fields.em0, i_edge, 3)); + } else { + Kokkos::parallel_for( + "WallEdgeE", + range, + kernel::hybrid::WallEdgeE_kernel(domain.fields.em0, i_edge, 3)); + } + }); + } + + /** + * @brief Even-mirror the wall ghosts of the predictor face-B scratch + * (cur = Bf*, Bf**). Call after each Faraday #1/#2 (+ halo + * exchange), before the EMF that reads it: the Hall stencil reads + * the wall ghosts, which no halo exchange fills (left at zero they + * fake a wall current sheet J ~ B/dx every substage). + */ + template + void WallScratchB(Domain>& domain, + const Grid& global_grid) { + wall::ForEachX1Wall(domain, global_grid, [&](short sign, ncells_t i_edge) { + const auto range = wall::StaggeredRange(domain, sign); + if (sign > 0) { + Kokkos::parallel_for( + "WallFaceB", + range, + kernel::hybrid::WallFaceB_kernel(domain.fields.cur, + i_edge, + 0)); + } else { + Kokkos::parallel_for( + "WallFaceB", + range, + kernel::hybrid::WallFaceB_kernel(domain.fields.cur, + i_edge, + 0)); + } + }); + } + + /** + * @brief Conductor-sign mirror fill of the wall ghosts of the + * cell-centered gather fields (bckp: Ec' 0..2, Bc' 3..5). Call + * after each EMF #1/#2 (+ halo exchange), before the particle push + * that gathers from bckp. + */ + template + void WallBckpFill(Domain>& domain, + const Grid& global_grid) { + wall::ForEachX1Wall(domain, global_grid, [&](short sign, ncells_t i_edge) { + const auto range = wall::CellRange(domain, false); + if (sign > 0) { + Kokkos::parallel_for( + "WallBckp", + range, + kernel::hybrid::WallBckp_kernel(domain.fields.bckp, i_edge)); + } else { + Kokkos::parallel_for( + "WallBckp", + range, + kernel::hybrid::WallBckp_kernel(domain.fields.bckp, i_edge)); + } + }); + } + + /** + * @brief Image-plasma wall treatment of the deposited moments (aux). + * @param fold also fold the ghost deposit tails back into the active cells + * (V_x sign-flipped) -- exactly once per deposit, after + * SynchronizeFields(AUX) + CommunicateFields(AUX). With + * fold = false only the (idempotent) ghost mirror fill runs, e.g. + * to re-fill after a filter pass. + */ + template + void MomentsWallBC(Domain>& domain, + const Grid& global_grid, + bool fold) { + wall::ForEachX1Wall(domain, global_grid, [&](short sign, ncells_t i_edge) { + if (fold) { + const auto fold_range = wall::CellRange(domain, true); + if (sign > 0) { + Kokkos::parallel_for("WallMomentsFold", + fold_range, + kernel::hybrid::WallMoments_kernel( + domain.fields.aux, + i_edge)); + } else { + Kokkos::parallel_for("WallMomentsFold", + fold_range, + kernel::hybrid::WallMoments_kernel( + domain.fields.aux, + i_edge)); + } + } + const auto fill_range = wall::CellRange(domain, false); + if (sign > 0) { + Kokkos::parallel_for("WallMomentsFill", + fill_range, + kernel::hybrid::WallMoments_kernel( + domain.fields.aux, + i_edge)); + } else { + Kokkos::parallel_for("WallMomentsFill", + fill_range, + kernel::hybrid::WallMoments_kernel( + domain.fields.aux, + i_edge)); + } + }); + } + + } // namespace hybrid +} // namespace ntt + +#endif // ENGINES_HYBRID_FIELDS_BCS_H diff --git a/src/engines/hybrid/fieldsolvers.h b/src/engines/hybrid/fieldsolvers.h new file mode 100644 index 000000000..6a27859a6 --- /dev/null +++ b/src/engines/hybrid/fieldsolvers.h @@ -0,0 +1,352 @@ +#ifndef ENGINES_HYBRID_FIELDSOLVERS_H +#define ENGINES_HYBRID_FIELDSOLVERS_H + +#include "enums.h" +#include "global.h" + +#include "utils/error.h" +#include "utils/param_container.h" + +#include "metrics/minkowski.h" + +#include "engines/hybrid/fields_bcs.h" +#include "framework/domain/domain.h" +#include "framework/domain/metadomain.h" +#include "framework/parameters/parameters.h" +#include "kernels/hybrid/EMF.hpp" +#include "kernels/hybrid/faraday.hpp" +#include "kernels/hybrid/subcycle.hpp" + +#include + +#if defined(MPI_ENABLED) + #include "arch/mpi_aliases.h" + + #include +#endif + +namespace ntt { + namespace hybrid { + + enum class faraday : uint8_t { + push1, + push2, + push3, + }; + + enum class emf : uint8_t { + push0, + push12, + }; + + template + void Faraday(Domain>& domain, + const prm::Parameters& engine_params, + const faraday& flag) { + const auto dt = engine_params.get("dt"); + const auto dx = domain.mesh.metric.get_dx(); + if (flag == faraday::push1) { + Kokkos::parallel_for( + "FaradayPush1", + domain.mesh.rangeActiveCells(), + kernel::hybrid::Faraday_kernel(domain.fields.em, + domain.fields.em, + domain.fields.cur, + 0, + 3, + 0, + dt, + dx)); + } else if (flag == faraday::push2) { + Kokkos::parallel_for( + "FaradayPush2", + domain.mesh.rangeActiveCells(), + kernel::hybrid::Faraday_kernel(domain.fields.em0, + domain.fields.em, + domain.fields.cur, + 3, + 3, + 0, + dt, + dx)); + } else if (flag == faraday::push3) { + Kokkos::parallel_for( + "FaradayPush3", + domain.mesh.rangeActiveCells(), + kernel::hybrid::Faraday_kernel(domain.fields.em0, + domain.fields.em, + domain.fields.em, + 3, + 3, + 3, + dt, + dx)); + } else { + raise::Error("Wrong option for `flag`", HERE); + } + } + + template + void EMF(Domain>& domain, + const prm::Parameters& engine_params, + const SimulationParams& params, + const emf& flag) { + const auto dt = engine_params.get("dt"); + const auto gamma_ad = params.get("hybrid.gamma_ad"); + const auto theta = params.get("hybrid.theta0"); + const auto d0 = params.get("scales.skindepth0"); + const auto rho0 = params.get("scales.larmor0"); + const auto dens_min = params.get("hybrid.dens_min"); + const auto hall_lim = params.get("hybrid.hall_lim"); + const auto res_vac = params.get("hybrid.resist_vac"); + const auto res_hyp = params.get("hybrid.resist_hyper"); + const auto dx = domain.mesh.metric.get_dx(); + if (flag == emf::push0) { + // clang-format off + Kokkos::parallel_for( + "EMFPush", + domain.mesh.rangeActiveCells(), + kernel::hybrid::EMF_kernel(domain.fields.aux, // P + domain.fields.aux, // N + domain.fields.em0, // Ee_in + domain.fields.cur, // Bf + domain.fields.bckp, // Ec + domain.fields.em, // Bfs + domain.fields.em, // Ee_out + domain.fields.em0, // Ec_out + domain.fields.bckp, // Bc_out + 0, 3, 3, // P, N, Ee_in + 0, 0, 3, // Bf, Ec, Bfs + 0, 0, 3, // Ee_out, Ec_out, Bc_out + dt, gamma_ad, theta, d0, rho0, dens_min, hall_lim, res_vac, res_hyp, dx)); + // clang-format on + } else { + // clang-format off + Kokkos::parallel_for( + "EMFPush", + domain.mesh.rangeActiveCells(), + kernel::hybrid::EMF_kernel(domain.fields.aux, // P + domain.fields.aux, // N + domain.fields.em, // Ee_in + domain.fields.em, // Bf + domain.fields.em0, // Ec + domain.fields.cur, // Bfs + domain.fields.em0, // Ee_out + domain.fields.bckp, // Ec_out + domain.fields.bckp, // Bc_out + 0, 3, 0, // P, N, Ee_in + 3, 0, 0, // Bf, Ec, Bfs + 3, 0, 3, // Ee_out, Ec_out, Bc_out + dt, gamma_ad, theta, d0, rho0, dens_min, hall_lim, res_vac, res_hyp, dx)); + // clang-format on + } + } + + /** + * @brief Sub-cycled (Pegasus-style) magnetic-field advance: integrates + * dB/dt = -curl E(B) over one full timestep with the Ohm's-law E + * refreshed from the (frozen) deposited moments every sub-step, so + * the whistler-stiff part of hybrid Ohm's law is integrated at its + * own CFL instead of the particle dt. + * + * Reads Bf^(n) from em::345 and the frozen moments from aux; leaves the + * advanced B (with valid ghosts) in `cur` -- the same contract as the + * single-Euler Faraday push #1/#2 it replaces. The third (accepting) + * Faraday push is redundant here: this advance already integrates the full + * interval, so the caller accepts `cur` as B^(n+1) (AcceptSubcycledB). + * + * Number of sub-steps m: adaptive from the GLOBAL max of the local whistler + * speed v_w = d0^2 pi |B| / (dx max(N, dens_min)), targeting a per-sub-step + * whistler Courant `hybrid.subcycle_courant`, capped at `hybrid.subcycle_max` + * (beyond the cap the per-cell Hall limiter, which here receives the + * SUB-step dt, takes over). The max is MPI-allreduced so every rank runs the + * same m and the halo exchanges inside the loop pair up. + * + * Each sub-step is SSP-RK3 (Shu-Osher): 2-stage schemes are marginally + * unstable for purely oscillatory (whistler) modes (|G| = 1 + (w dt)^4/8), + * while RK3 is damped up to w dt < sqrt(3). + * + * Scratch usage: cur = running B (in place), buff = sub-step base for the + * RK3 combinations, em0::345 = stage E. Combines run over the FULL extent so + * valid ghosts stay valid; E/B halo exchanges + wall conditions run after + * every stage. + */ + template + auto SubcycledFaraday(Metadomain>& metadomain, + Domain>& domain, + const prm::Parameters& engine_params, + const SimulationParams& params) -> int { + const auto dt = engine_params.get("dt"); + const auto dx = domain.mesh.metric.get_dx(); + const auto gamma_ad = params.get("hybrid.gamma_ad"); + const auto theta = params.get("hybrid.theta0"); + const auto d0 = params.get("scales.skindepth0"); + const auto rho0 = params.get("scales.larmor0"); + const auto dens_min = params.get("hybrid.dens_min"); + const auto hall_lim = params.get("hybrid.hall_lim"); + const auto res_vac = params.get("hybrid.resist_vac"); + const auto res_hyp = params.get("hybrid.resist_hyper"); + const auto courant = params.get("hybrid.subcycle_courant"); + auto m_max = params.get("hybrid.subcycle_max"); + raise::ErrorIf(courant <= ZERO, + "hybrid.subcycle_courant must be positive", + HERE); + if (m_max < 1) { + m_max = 1; + } + + // pick m from the global max whistler speed at the advance start + real_t vw_max = ZERO; + Kokkos::parallel_reduce("VwMax", + domain.mesh.rangeActiveCells(), + kernel::hybrid::VwMax_kernel(domain.fields.em, + domain.fields.aux, + 3, + 3, + d0, + dens_min, + dx), + Kokkos::Max(vw_max)); +#if defined(MPI_ENABLED) + MPI_Allreduce(MPI_IN_PLACE, + &vw_max, + 1, + mpi::get_type(), + MPI_MAX, + MPI_COMM_WORLD); +#endif + // clamp in double before the int cast: at runaway field amplitudes + // vw_max*dt/(courant*dx) exceeds INT_MAX, and casting that to int is UB + // (wraps negative), which the m < 1 -> m = 1 clamp would then turn into + // m = 1, disabling the subcycler exactly when it is needed. Comparing in + // double and taking the else-branch also routes a NaN/Inf vw_max to m_max. + const double m_want = std::ceil(static_cast(vw_max) * dt / + (courant * dx)); + int m; + if (m_want < static_cast(m_max)) { + m = static_cast(m_want); + if (m < 1) { + m = 1; + } + } else { + m = m_max; + } + const real_t dt_sub = dt / static_cast(m); + + const auto range_act = domain.mesh.rangeActiveCells(); + const auto range_all = domain.mesh.rangeAllCells(); + + // stage E from frozen moments and the current sub-stepped B (cur): + // raw edge-E only -> em0::345 (+ halo + wall condition). The Hall limiter + // inside receives the SUB-step dt. + auto emf_stage = [&]() { + // clang-format off + Kokkos::parallel_for( + "EMFSub", + range_act, + kernel::hybrid::EMF_kernel(domain.fields.aux, // P + domain.fields.aux, // N + domain.fields.em0, // Ee_in (unused) + domain.fields.cur, // Bf (unused) + domain.fields.em0, // Ec (unused) + domain.fields.cur, // Bfs = B_s + domain.fields.em0, // Ee_out + domain.fields.em0, // Ec_out (unused) + domain.fields.bckp, // Bc_out (unused) + 0, 3, 0, // P, N, Ee_in + 0, 0, 0, // Bf, Ec, Bfs + 3, 0, 0, // Ee_out, Ec_out, Bc_out + dt_sub, gamma_ad, theta, d0, rho0, dens_min, hall_lim, res_vac, res_hyp, dx)); + // clang-format on + metadomain.CommunicateFields(domain, ::Comm::EM0_345); + WallEPrime(domain, metadomain.mesh()); + }; + // cur += dt_sub * curl(em0::345), in place (+ halo + wall condition) + auto faraday_stage = [&]() { + Kokkos::parallel_for( + "FaradaySub", + range_act, + kernel::hybrid::Faraday_kernel(domain.fields.em0, + domain.fields.cur, + domain.fields.cur, + 3, + 0, + 0, + dt_sub, + dx)); + metadomain.CommunicateFields(domain, ::Comm::CUR); + WallScratchB(domain, metadomain.mesh()); + }; + // cur = a*cur + b*buff over the full extent (ghosts of both are valid) + auto combine = [&](real_t a, real_t b) { + Kokkos::parallel_for( + "FieldCombine", + range_all, + kernel::hybrid::FieldCombine_kernel(domain.fields.cur, + domain.fields.buff, + a, + b, + 0, + 0)); + }; + + // start from Bf^(n), ghosts included (valid: comm'd + wall'd at the end + // of the previous step) + Kokkos::parallel_for( + "FieldCombine", + range_all, + kernel::hybrid::FieldCombine_kernel(domain.fields.cur, + domain.fields.em, + ZERO, + ONE, + 0, + 3)); + for (int s = 0; s < m; ++s) { + // sub-step base: buff <- cur + Kokkos::parallel_for( + "FieldCombine", + range_all, + kernel::hybrid::FieldCombine_kernel(domain.fields.buff, + domain.fields.cur, + ZERO, + ONE, + 0, + 0)); + // SSP-RK3 (Shu-Osher): u1 = u + dt L(u) + emf_stage(); + faraday_stage(); + // u2 = 3/4 u + 1/4 (u1 + dt L(u1)) + emf_stage(); + faraday_stage(); + combine(static_cast(0.25), static_cast(0.75)); + // u^(s+1) = 1/3 u + 2/3 (u2 + dt L(u2)) + emf_stage(); + faraday_stage(); + combine(static_cast(2.0 / 3.0), static_cast(1.0 / 3.0)); + } + return m; + } + + /** + * @brief Accept the sub-cycled advance as B^(n+1): em::345 <- cur (full + * extent -- the ghosts of cur are valid). Replaces Faraday push #3, + * which in sub-cycled mode would redundantly re-integrate the same + * interval with a stiff-unstable single Euler step. + */ + template + void AcceptSubcycledB(Domain>& domain) { + Kokkos::parallel_for( + "FieldCombine", + domain.mesh.rangeAllCells(), + kernel::hybrid::FieldCombine_kernel(domain.fields.em, + domain.fields.cur, + ZERO, + ONE, + 3, + 0)); + } + + } // namespace hybrid +} // namespace ntt + +#endif diff --git a/src/engines/hybrid/hybrid.hpp b/src/engines/hybrid/hybrid.hpp new file mode 100644 index 000000000..a29a1a92a --- /dev/null +++ b/src/engines/hybrid/hybrid.hpp @@ -0,0 +1,326 @@ +#ifndef ENGINES_HYBRID_HYBRID_HPP +#define ENGINES_HYBRID_HYBRID_HPP + +#include "enums.h" + +#include "traits/metric.h" +#include "utils/timer.h" + +#include "engines/hybrid/fields_bcs.h" +#include "engines/hybrid/fieldsolvers.h" +#include "engines/hybrid/moments_filter.h" +#include "engines/hybrid/particle_pusher.h" + +#include "engines/engine.hpp" + +namespace ntt { + + template + class HYBRIDEngine : public Engine { + using base_t = Engine; + using pgen_t = user::PGen; + using domain_t = Domain; + // contents + using base_t::m_metadomain; + using base_t::m_params; + using base_t::m_pgen; + // methods + using base_t::init; + // variables + using base_t::dt; + using base_t::max_steps; + using base_t::runtime; + using base_t::step; + using base_t::time; + + public: + static constexpr auto S { SimEngine::HYBRID }; + + HYBRIDEngine(const SimulationParams& params) : base_t { params } {} + + ~HYBRIDEngine() override = default; + + void step_forward(timer::Timers& timers, domain_t& dom) override { + /** + * Initially: em::012 -- + * em::345 Bf^(n) + * + * em0::012 -- + * em0::345 -- + * + * aux::012 V^(n) (except step 0) + * aux::3 N^(n) (except step 0) + * + * bckp::012 -- + * bckp::345 -- + * + * cur::012 -- + * + * x_prtl at n + * u_prtl at n + */ + // Pegasus-style sub-cycled field advance (fieldsolvers.h::SubcycledFaraday): + // the whistler-stiff Ohm's-law/Faraday loop is integrated with adaptive + // SSP-RK3 sub-steps at its own CFL instead of single full-dt Euler pushes. + // false -> the legacy 3-push scheme. + const bool subcycle = m_params.template get("hybrid.subcycle"); + + if (step == 0) { + // fill Bf^(n) ghosts (periodic / MPI) so the field-solver stencils are valid + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, ::Comm::EM_345); + timers.stop("Communications"); + // non-periodic (reflecting-wall) boundaries on the initial Bf, if any + timers.start("FieldBoundaries"); + hybrid::FieldBoundaries(dom, m_metadomain.mesh(), BC::B); + timers.stop("FieldBoundaries"); + // compute N^(0) and V^(0) -> aux::012, aux::3 (deposit-only, no push) + timers.start("Moments"); + hybrid::DepositMoments(dom, m_params); + m_metadomain.SynchronizeFields(dom, ::Comm::AUX); // additive remap of deposit tails + m_metadomain.CommunicateFields(dom, ::Comm::AUX); // fill ghosts for EMF reads + // reflecting wall: fold the wall-ghost deposit tails back (image plasma) + // + mirror-fill the wall ghosts for the EMF stencils + hybrid::MomentsWallBC(dom, m_metadomain.mesh(), /* fold */ true); + // smooth N^(0),V^(0) (kills grid-scale shot noise before the field solve) + hybrid::MomentsFilter(m_metadomain, dom, m_params); + timers.stop("Moments"); + } + + // EMF calculation #0 โ€” the un-averaged field E^(n). + // Using: aux::012 [V^(n)], aux::3 [N^(n)], em::345 [Bf^(n)] + // + // Ee^(n) = EMF(N^(n), V^(n), Bf^(n)) -> em::012 + // Ec^(n) = EMF(N^(n), V^(n), Bc^(n)=interp Bf) -> em0::012 + // (these seed the trapezoidal average E' = 1/2(E* + E^(n)) in EMF #1/#2) + timers.start("FieldSolver"); + hybrid::EMF(dom, this->engineParams(), m_params, hybrid::emf::push0); + timers.stop("FieldSolver"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, ::Comm::EM_012 | ::Comm::EM0_012); + timers.stop("Communications"); + // reflecting wall: E_tan = 0 on the wall plane of Ee^(n) before Faraday #1 + // (this is what freezes the wall-plane B_n for an oblique background field) + timers.start("FieldBoundaries"); + hybrid::FieldBoundaries(dom, m_metadomain.mesh(), BC::E); + timers.stop("FieldBoundaries"); + + // Faraday push #1 (predictor field advance) + // Using: em::012 [Ee^(n)], em::345 [Bf^(n)] + // + // legacy: Bf* = Bf^(n) + dt * curl Ee^(n) + // subcycled: Bf* = integrate dB/dt = -curl EMF(N^(n), V^(n), B) over dt + // (E refreshed every sub-step; comms + wall conditions inside, + // cur comes back with valid ghosts) + // + // Now: cur::012 <-- Bf* + timers.start("FieldSolver"); + if (subcycle) { + hybrid::SubcycledFaraday(m_metadomain, dom, this->engineParams(), m_params); + } else { + hybrid::Faraday(dom, this->engineParams(), hybrid::faraday::push1); + } + timers.stop("FieldSolver"); + if (not subcycle) { + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, ::Comm::CUR); // fill Bf* ghosts for EMF #1 + timers.stop("Communications"); + // reflecting wall: even-mirror the Bf* scratch ghosts (a physical wall + // has no halo exchange to fill them) so the Hall stencil sees no fake + // wall current + timers.start("FieldBoundaries"); + hybrid::WallScratchB(dom, m_metadomain.mesh()); + timers.stop("FieldBoundaries"); + } + + // EMF calculation #1 + // Using: + // aux::012 [P^(n)] + // aux::3 [N^(n)] + // em::012 [Ee^(n)] + // em::345 [Bf^(n)] + // em0::012 [Ec^(n)] + // cur::012 [Bf*] + // + // Bc* = interpolate Bf* + // Bc^(n) = interpolate Bf^(n) + // Ee* = EMF(N^(n), P^(n), Bf*) + // Ec* = EMF(N^(n), P^(n), Bc*) + // + // Ee' = 0.5 * (Ee* + Ee^(n)) + // Ec' = 0.5 * (Ec* + Ec^(n)) + // Bc' = 0.5 * (Bc* + Bc^(n)) + // + // Now: + // em0::345 <-- Ee' + // bckp::012 <-- Ec' + // bckp::345 <-- Bc' + timers.start("FieldSolver"); + hybrid::EMF(dom, this->engineParams(), m_params, hybrid::emf::push12); + timers.stop("FieldSolver"); + + // Particle push #1 (predictor) โ€” Pegasus Fig. 2 steps 7+8. + // Using: bckp::012 [Ec'], bckp::345 [Bc']; transient (no store, no particle BCs). + // Pushes in registers and deposits predicted moments: aux::012 <-- V', aux::3 <-- N' + timers.start("Communications"); + // Ee' (em0::345) ghosts for Faraday push #2; Ec'/Bc' (bckp) ghosts for the gather + m_metadomain.CommunicateFields(dom, ::Comm::EM0_345 | ::Comm::Bckp); + timers.stop("Communications"); + // reflecting wall: E_tan = 0 on the wall plane of Ee' (consumed by + // Faraday #2) + conductor-mirror the bckp wall ghosts for the gather + timers.start("FieldBoundaries"); + hybrid::WallEPrime(dom, m_metadomain.mesh()); + hybrid::WallBckpFill(dom, m_metadomain.mesh()); + timers.stop("FieldBoundaries"); + timers.start("ParticlePusher"); + hybrid::ParticlePush(dom, this->engineParams(), m_params, /* corrector */ false); + timers.stop("ParticlePusher"); + timers.start("Moments"); + m_metadomain.SynchronizeFields(dom, + ::Comm::AUX); // additive remap of deposit tails + m_metadomain.CommunicateFields(dom, ::Comm::AUX); // fill ghosts for EMF #2 + // reflecting wall: image-plasma fold + ghost fill of the predicted moments + hybrid::MomentsWallBC(dom, m_metadomain.mesh(), /* fold */ true); + // smooth predicted N',V' before EMF #2 + hybrid::MomentsFilter(m_metadomain, dom, m_params); + timers.stop("Moments"); + + // Faraday push #2 (corrector field advance) + // Using: em::345 [Bf^(n)] + em0::345 [Ee'] (legacy) / aux [N', V'] (subcycled) + // + // legacy: Bf** = Bf^(n) + dt * curl Ee' + // subcycled: Bf** = integrate dB/dt = -curl EMF(N', V', B) over dt + // (E refreshed from the PREDICTED moments every sub-step; + // this advance IS the full-interval B^(n) -> B^(n+1) + // integration, accepted as Bf^(n+1) below) + // + // Now: cur::012 <-- Bf** + timers.start("FieldSolver"); + if (subcycle) { + hybrid::SubcycledFaraday(m_metadomain, dom, this->engineParams(), m_params); + } else { + hybrid::Faraday(dom, this->engineParams(), hybrid::faraday::push2); + } + timers.stop("FieldSolver"); + if (not subcycle) { + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, ::Comm::CUR); // fill Bf** ghosts for EMF #2 + timers.stop("Communications"); + // reflecting wall: even-mirror the Bf** scratch ghosts (see Faraday #1) + timers.start("FieldBoundaries"); + hybrid::WallScratchB(dom, m_metadomain.mesh()); + timers.stop("FieldBoundaries"); + } + + // EMF calculation #2 + // Using: + // aux::012 [P'] + // aux::3 [N'] + // em::012 [Ee^(n)] + // em::345 [Bf^(n)] + // em0::012 [Ec^(n)] + // cur::012 [Bf**] + // + // Bc** = interpolate Bf** + // Bc^(n) = interpolate Bf^(n) + // Ee** = EMF(N', P', Bf**) + // Ec** = EMF(N', P', Bc**) + // + // Ee'' = 0.5 * (Ee** + Ee^(n)) + // Ec'' = 0.5 * (Ec** + Ec^(n)) + // Bc'' = 0.5 * (Bc** + Bc^(n)) + // + // Now: + // em0::345 <-- Ee'' + // bckp:012 <-- Ec'' + // bckp:345 <-- Bc'' + timers.start("FieldSolver"); + hybrid::EMF(dom, this->engineParams(), m_params, hybrid::emf::push12); + timers.stop("FieldSolver"); + timers.start("Communications"); + // Ee'' (em0::345) ghosts for Faraday push #3; Ec''/Bc'' (bckp) ghosts for the gather + m_metadomain.CommunicateFields(dom, ::Comm::EM0_345 | ::Comm::Bckp); + timers.stop("Communications"); + // reflecting wall: E_tan = 0 on the wall plane of Ee'' (consumed by + // Faraday #3) + conductor-mirror the bckp wall ghosts for the corrector + timers.start("FieldBoundaries"); + hybrid::WallEPrime(dom, m_metadomain.mesh()); + hybrid::WallBckpFill(dom, m_metadomain.mesh()); + timers.stop("FieldBoundaries"); + + // Faraday push #3 (accept B^(n+1)) + // Using: em0::345 [Ee''], em::345 [Bf^(n)] (legacy) / cur [Bf**] (subcycled) + // + // legacy: Bf^(n+1) = Bf^(n) + dt * curl Ee'' + // subcycled: Bf^(n+1) = Bf** -- the corrector advance above already + // integrated the full interval; a third full-dt Euler push + // would redundantly re-integrate it (stiff-unstable) + // + // Now: + // em::345 <-- Bf^(n+1) + timers.start("FieldSolver"); + if (subcycle) { + hybrid::AcceptSubcycledB(dom); + } else { + hybrid::Faraday(dom, this->engineParams(), hybrid::faraday::push3); + } + timers.stop("FieldSolver"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, ::Comm::EM_345); // Bf^(n+1) ghosts for next step + timers.stop("Communications"); + // non-periodic (reflecting-wall) boundaries on Bf^(n+1), if any + timers.start("FieldBoundaries"); + hybrid::FieldBoundaries(dom, m_metadomain.mesh(), BC::B); + timers.stop("FieldBoundaries"); + + // Particle push #2 (corrector) โ€” Pegasus Fig. 2 step 12. + // Using: bckp::012 [Ec''], bckp::345 [Bc'']; accepted (store-back + + // particle BCs). Pushes and deposits final moments: x_prtl at n+1, + // aux::012 <-- V^(n+1), aux::3 <-- N^(n+1) (bckp ghosts already filled + // after EMF #2; Faraday push #3 does not touch bckp) + timers.start("ParticlePusher"); + hybrid::ParticlePush(dom, this->engineParams(), m_params, /* corrector */ true); + timers.stop("ParticlePusher"); + + timers.start("Moments"); + m_metadomain.SynchronizeFields( + dom, + ::Comm::AUX); // additive remap of deposit tails (pre-migration) + m_metadomain.CommunicateFields(dom, ::Comm::AUX); // fill ghosts for next step's EMF + // reflecting wall: image-plasma fold + ghost fill of the final moments + hybrid::MomentsWallBC(dom, m_metadomain.mesh(), /* fold */ true); + // smooth final N^(n+1),V^(n+1) for next step's EMF #0/#1 + hybrid::MomentsFilter(m_metadomain, dom, m_params); + timers.stop("Moments"); + + timers.start("Communications"); + m_metadomain.CommunicateParticles(dom); + timers.stop("Communications"); + + timers.start("ParticleSort"); + m_metadomain.SortParticles(time, step, m_params, dom); + timers.stop("ParticleSort"); + + /** + * Finally: em::012 -- + * em::345 Bf^(n+1) + * + * em0::012 -- + * em0::345 -- + * + * aux::012 V^(n+1) + * aux::3 N^(n+1) + * + * bckp::012 -- + * bckp::345 -- + * + * cur::012 -- + * + * x_prtl at n+1 + * u_prtl at n+1 + */ + } + }; +} // namespace ntt + +#endif // ENGINES_HYBRID_HYBRID_HPP diff --git a/src/engines/hybrid/moments_filter.h b/src/engines/hybrid/moments_filter.h new file mode 100644 index 000000000..2e2646878 --- /dev/null +++ b/src/engines/hybrid/moments_filter.h @@ -0,0 +1,62 @@ +#ifndef ENGINES_HYBRID_MOMENTS_FILTER_H +#define ENGINES_HYBRID_MOMENTS_FILTER_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/log.h" +#include "utils/param_container.h" + +#include "metrics/minkowski.h" + +#include "engines/hybrid/fields_bcs.h" +#include "framework/domain/domain.h" +#include "framework/domain/metadomain.h" +#include "framework/parameters/parameters.h" +#include "kernels/hybrid/moments_filter.hpp" + +namespace ntt { + namespace hybrid { + + /** + * Binomially smooth the deposited moments (aux::012 = V, aux::3 = N) in + * place, `algorithms.current_filters` times. Without this, cold/fast beams + * inject grid-scale shot noise straight into the Ohm's-law E and drive a + * numerical instability (see kernels/hybrid/moments_filter.hpp). + * + * Must be called AFTER the deposit's additive ghost remap + * (SynchronizeFields(AUX)) and active->ghost copy (CommunicateFields(AUX)), + * since the stencil reads the i +/- 1 ghosts; each pass re-fills them. + * + * Uses `bckp` as the read buffer: it is dead scratch at every deposit point + * in the step (always overwritten by the following EMF before it is read). + */ + template + void MomentsFilter(Metadomain>& metadomain, + Domain>& domain, + const SimulationParams& params) { + const auto nfilter = params.template get( + "algorithms.current_filters"); + if (nfilter == 0) { + return; + } + logger::Checkpoint("Launching hybrid moments filtering kernels", HERE); + for (auto i { 0u }; i < nfilter; ++i) { + Kokkos::deep_copy(domain.fields.bckp, domain.fields.aux); + Kokkos::parallel_for( + "MomentsFilter", + domain.mesh.rangeActiveCells(), + kernel::hybrid::MomentsFilter_kernel(domain.fields.aux, + domain.fields.bckp)); + metadomain.CommunicateFields(domain, ::Comm::AUX); + // re-mirror the reflecting-wall ghosts (fill only; the deposit tails + // were already folded once, before the filter) + MomentsWallBC(domain, metadomain.mesh(), /* fold */ false); + } + } + + } // namespace hybrid +} // namespace ntt + +#endif // ENGINES_HYBRID_MOMENTS_FILTER_H diff --git a/src/engines/hybrid/particle_pusher.h b/src/engines/hybrid/particle_pusher.h new file mode 100644 index 000000000..266c192ec --- /dev/null +++ b/src/engines/hybrid/particle_pusher.h @@ -0,0 +1,139 @@ +/** + * @file engines/hybrid/particle_pusher.h + * @brief Particle push + fused ion-moment deposit drivers for the HYBRID engine + * @implements + * - ntt::hybrid::ParticlePush<> -> void + * - ntt::hybrid::DepositMoments<> -> void + * @namespaces: + * - ntt::hybrid:: + * + * The Pegasus step (Kunz, Stone & Bai 2014, Fig. 2) performs two ion pushes that + * BOTH start from the stored state x^(n), v^(n). The pusher kernel deposits the + * ion moments (N, V) in the SAME pass as the push, so the transient predictor can + * produce its predicted moments without ever writing the particle arrays โ€” hence + * no save/restore of x^(n), v^(n) is needed: + * + * DepositMoments(dom, params); // step 0 only: seed aux with N^(0), V^(0) + * ...EMF #1... + * ParticlePush(dom, ep, params, false); // predictor: push (registers) + deposit N', V' + * ...EMF #2, Faraday... + * ParticlePush(dom, ep, params, true); // corrector: push + deposit + store x^(n+1) + * + * Each call rebuilds the `aux` ScatterView, zeroes aux, runs the kernel (which + * scatter-deposits), and contributes. The caller then remaps/fills aux ghosts: + * SynchronizeFields(dom, ::Comm::AUX) // additive ghost->active (Pegasus ยง3.6) + * CommunicateFields(dom, ::Comm::AUX) // copy active->ghost for the EMF reads + * and, before each push, fills the bckp (Ec/Bc) ghosts the gather reads: + * CommunicateFields(dom, ::Comm::Bckp) + * + * @see kernels/hybrid/pusher.hpp and PIC/hybrid/pusher.md. + */ + +#ifndef ENGINES_HYBRID_PARTICLE_PUSHER_H +#define ENGINES_HYBRID_PARTICLE_PUSHER_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "traits/metric.h" +#include "utils/comparators.h" +#include "utils/log.h" + +#include "framework/domain/domain.h" +#include "framework/parameters/parameters.h" +#include "kernels/hybrid/pusher.hpp" +#include "kernels/pushers/context.h" // kernel::sr::PusherBoundaries + +#include +#include + +namespace ntt::hybrid { + + /** + * @brief Run the fused push+deposit kernel over all ion species into `aux`. + * Zeroes aux, scatter-deposits N -> aux::3 and V = ฮฃ m v -> aux::0..2, + * and contributes. Caller handles the subsequent ghost sync/comm. + * @tparam Mode MomentsOnly (no push), Predictor (no store), Corrector (store). + * @param dt time-step (unused for MomentsOnly). + */ + template + void runPusher(Domain& domain, + const SimulationParams& params, + real_t dt) { + const auto omegaB0 = params.template get("scales.omegaB0"); + const auto inv_n0 = ONE / params.template get("scales.n0"); + const auto use_weights = params.template get("particles.use_weights"); + + const auto pusher_boundaries = kernel::sr::PusherBoundaries { + domain.mesh.prtl_bc() + }; + + Kokkos::deep_copy(domain.fields.aux, ZERO); + auto scatter_aux = Kokkos::Experimental::create_scatter_view(domain.fields.aux); + + for (auto& species : domain.species) { + if ((species.npart() == 0) or cmp::AlmostZero_host(species.mass())) { + continue; + } + if constexpr (Mode != kernel::hybrid::PushMode::MomentsOnly) { + species.set_unsorted(); + } + + const kernel::hybrid::PusherContext ctx { + species.mass(), + species.charge(), + dt, + omegaB0, + inv_n0, + use_weights, + static_cast(domain.mesh.n_active(in::x1)), + static_cast(domain.mesh.n_active(in::x2)), + static_cast(domain.mesh.n_active(in::x3)) + }; + + Kokkos::parallel_for( + "HybridPushDeposit", + species.rangeActiveParticles(), + kernel::hybrid::Pusher_kernel { ctx, + pusher_boundaries, + species, + domain.fields.bckp, + scatter_aux, + domain.mesh.metric }); + } + Kokkos::Experimental::contribute(domain.fields.aux, scatter_aux); + } + + /** + * @brief Advance all ion species from x^(n),v^(n) to x^(n+1),v^(n+1) and deposit + * the corresponding ion moments into `aux` in the same pass. + * @param corrector false -> transient predictor push (no store, no particle BCs); + * true -> accepted corrector push (store-back + particle BCs). + */ + template + void ParticlePush(Domain& domain, + const prm::Parameters& engine_params, + const SimulationParams& params, + bool corrector) { + const auto dt = engine_params.get("dt"); + if (corrector) { + runPusher(domain, params, dt); + } else { + runPusher(domain, params, dt); + } + } + + /** + * @brief Deposit ion moments N, V from the stored particles into `aux` without + * pushing. Used once at step 0 to seed aux with N^(0), V^(0). + */ + template + void DepositMoments(Domain& domain, + const SimulationParams& params) { + runPusher(domain, params, ZERO); + } + +} // namespace ntt::hybrid + +#endif // ENGINES_HYBRID_PARTICLE_PUSHER_H diff --git a/src/engines/srpic/particles_bcs.h b/src/engines/srpic/particles_bcs.h index 5e79f3b78..eab19b63f 100644 --- a/src/engines/srpic/particles_bcs.h +++ b/src/engines/srpic/particles_bcs.h @@ -105,7 +105,7 @@ namespace ntt { prtl_spec.set_unsorted(); } Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); - metadomain.SynchronizeFields(domain, Comm::Bckp, { 0, 1 }); + metadomain.SynchronizeFields(domain, ::Comm::Bckp, { 0, 1 }); } const auto maxwellian = arch::energy_dist::Maxwellian { diff --git a/src/engines/srpic/srpic.hpp b/src/engines/srpic/srpic.hpp index d0352318e..4a48ff1c6 100644 --- a/src/engines/srpic/srpic.hpp +++ b/src/engines/srpic/srpic.hpp @@ -36,6 +36,12 @@ namespace ntt { + namespace srpic::Comm { + static constexpr auto E = ::Comm::EM_012; + static constexpr auto B = ::Comm::EM_345; + static constexpr auto J = ::Comm::CUR; + } // namespace srpic::Comm + template class SRPICEngine : public Engine { @@ -70,7 +76,7 @@ namespace ntt { if (step == 0) { // communicate fields and apply BCs on the first timestep - m_metadomain.CommunicateFields(dom, Comm::B | Comm::E); + m_metadomain.CommunicateFields(dom, srpic::Comm::B | srpic::Comm::E); srpic::FieldBoundaries(dom, m_metadomain.mesh().metric, m_metadomain.mesh(), @@ -87,7 +93,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B); + m_metadomain.CommunicateFields(dom, srpic::Comm::B); timers.stop("Communications"); timers.start("FieldBoundaries"); @@ -118,8 +124,8 @@ namespace ntt { timers.stop("CurrentDeposit"); timers.start("Communications"); - m_metadomain.SynchronizeFields(dom, Comm::J); - m_metadomain.CommunicateFields(dom, Comm::J); + m_metadomain.SynchronizeFields(dom, srpic::Comm::J); + m_metadomain.CommunicateFields(dom, srpic::Comm::J); timers.stop("Communications"); timers.start("CurrentFiltering"); @@ -138,7 +144,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B); + m_metadomain.CommunicateFields(dom, srpic::Comm::B); timers.stop("Communications"); timers.start("FieldBoundaries"); @@ -162,7 +168,7 @@ namespace ntt { } timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::E | Comm::J); + m_metadomain.CommunicateFields(dom, srpic::Comm::E | srpic::Comm::J); timers.stop("Communications"); timers.start("FieldBoundaries"); diff --git a/src/entity.cpp b/src/entity.cpp index c05c6c18a..b7a32600e 100644 --- a/src/entity.cpp +++ b/src/entity.cpp @@ -10,6 +10,7 @@ #include "framework/specialization_registry.h" #include "engines/grpic/grpic.hpp" +#include "engines/hybrid/hybrid.hpp" #include "engines/srpic/srpic.hpp" #include "pgen.hpp" @@ -32,6 +33,12 @@ namespace ntt { template using type = GRPICEngine; }; + + template <> + struct EngineSelector { + template + using type = HYBRIDEngine; + }; } // namespace ntt template @@ -48,9 +55,7 @@ static constexpr bool should_compile { template class M, Dimension D> void dispatch_engine(ntt::Simulation& sim) { - if constexpr (S == SimEngine::SRPIC) { - sim.run::template type, M, D>(); - } else if constexpr (S == SimEngine::GRPIC) { + if constexpr (S != ntt::SimEngine::INVALID) { sim.run::template type, M, D>(); } else { static_assert(::traits::always_false>::value, diff --git a/src/framework/containers/fields.cpp b/src/framework/containers/fields.cpp index 6d379ee34..8207e90b7 100644 --- a/src/framework/containers/fields.cpp +++ b/src/framework/containers/fields.cpp @@ -3,6 +3,10 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" + +#include "framework/specialization_registry.h" + #include namespace ntt { @@ -23,14 +27,27 @@ namespace ntt { bckp = ndfield_t { "BCKP", nx1 }; cur = ndfield_t { "J", nx1 }; buff = ndfield_t { "BUFF", nx1 }; + if constexpr (::traits::engine::DefinesAuxFields) { + aux = ndfield_t { "AUX", nx1 }; + } + if constexpr (::traits::engine::DefinesEM0Fields) { + em0 = ndfield_t { "EM0", nx1 }; + } + if constexpr (::traits::engine::DefinesCur0Fields) { + cur0 = ndfield_t { "CUR0", nx1 }; + } } else if constexpr (D == Dim::_2D) { em = ndfield_t { "EM", nx1, nx2 }; bckp = ndfield_t { "BCKP", nx1, nx2 }; cur = ndfield_t { "J", nx1, nx2 }; buff = ndfield_t { "BUFF", nx1, nx2 }; - if constexpr (S == SimEngine::GRPIC) { - aux = ndfield_t { "AUX", nx1, nx2 }; - em0 = ndfield_t { "EM0", nx1, nx2 }; + if constexpr (::traits::engine::DefinesAuxFields) { + aux = ndfield_t { "AUX", nx1, nx2 }; + } + if constexpr (::traits::engine::DefinesEM0Fields) { + em0 = ndfield_t { "EM0", nx1, nx2 }; + } + if constexpr (::traits::engine::DefinesCur0Fields) { cur0 = ndfield_t { "CUR0", nx1, nx2 }; } } else if constexpr (D == Dim::_3D) { @@ -38,18 +55,23 @@ namespace ntt { bckp = ndfield_t { "BCKP", nx1, nx2, nx3 }; cur = ndfield_t { "J", nx1, nx2, nx3 }; buff = ndfield_t { "BUFF", nx1, nx2, nx3 }; - if constexpr (S == SimEngine::GRPIC) { - aux = ndfield_t { "AUX", nx1, nx2, nx3 }; - em0 = ndfield_t { "EM0", nx1, nx2, nx3 }; + if constexpr (::traits::engine::DefinesAuxFields) { + aux = ndfield_t { "AUX", nx1, nx2, nx3 }; + } + if constexpr (::traits::engine::DefinesEM0Fields) { + em0 = ndfield_t { "EM0", nx1, nx2, nx3 }; + } + if constexpr (::traits::engine::DefinesCur0Fields) { cur0 = ndfield_t { "CUR0", nx1, nx2, nx3 }; } } } - template struct Fields; - template struct Fields; - template struct Fields; - template struct Fields; - template struct Fields; + // NOLINTBEGIN(bugprone-macro-parentheses) +#define FIELDS_CONSTRUCTOR(D, S) template struct Fields; + + NTT_FOREACH_SPECIALIZATION_FIELDS(FIELDS_CONSTRUCTOR) +#undef FIELDS_CONSTRUCTOR + // NOLINTEND(bugprone-macro-parentheses) } // namespace ntt diff --git a/src/framework/containers/fields_io.cpp b/src/framework/containers/fields_io.cpp index a17ab1430..2025ee8de 100644 --- a/src/framework/containers/fields_io.cpp +++ b/src/framework/containers/fields_io.cpp @@ -1,9 +1,11 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" #include "utils/log.h" #include "framework/containers/fields.h" +#include "framework/specialization_registry.h" #include "output/utils/readers.h" #include "output/utils/writers.h" @@ -30,8 +32,10 @@ namespace ntt { ls6.push_back(6); io.DefineVariable("em", gs6, lo6, ls6); - if (S == ntt::SimEngine::GRPIC) { + if (::traits::engine::WriteEM0FieldToCheckpoint) { io.DefineVariable("em0", gs6, lo6, ls6); + } + if (::traits::engine::WriteCurFieldToCheckpoint) { auto gs3 = std::vector(global_shape.begin(), global_shape.end()); auto lo3 = std::vector(local_offset.begin(), local_offset.end()); auto ls3 = std::vector(local_shape.begin(), local_shape.end()); @@ -52,8 +56,10 @@ namespace ntt { range6.first.push_back(0); range6.second.push_back(6); out::ReadNDField(io, reader, "em", em, range6); - if (S == ntt::SimEngine::GRPIC) { + if (::traits::engine::WriteEM0FieldToCheckpoint) { out::ReadNDField(io, reader, "em0", em0, range6); + } + if (::traits::engine::WriteCurFieldToCheckpoint) { auto range3 = adios2::Box(range.first, range.second); range3.first.push_back(0); range3.second.push_back(3); @@ -86,6 +92,7 @@ namespace ntt { } } + // NOLINTBEGIN(bugprone-macro-parentheses) #define FIELDS_CHECKPOINTS(D, S) \ template void Fields::CheckpointDeclare(adios2::IO&, \ const std::vector&, \ @@ -100,11 +107,8 @@ namespace ntt { const std::vector&, \ const std::vector&) const; - FIELDS_CHECKPOINTS(Dim::_1D, SimEngine::SRPIC) - FIELDS_CHECKPOINTS(Dim::_2D, SimEngine::SRPIC) - FIELDS_CHECKPOINTS(Dim::_3D, SimEngine::SRPIC) - FIELDS_CHECKPOINTS(Dim::_2D, SimEngine::GRPIC) - FIELDS_CHECKPOINTS(Dim::_3D, SimEngine::GRPIC) + NTT_FOREACH_SPECIALIZATION_FIELDS(FIELDS_CHECKPOINTS) #undef FIELDS_CHECKPOINTS + // NOLINTEND(bugprone-macro-parentheses) } // namespace ntt diff --git a/src/framework/domain/metadomain_comm.cpp b/src/framework/domain/metadomain_comm.cpp index f8c0f60d7..2cd558a0c 100644 --- a/src/framework/domain/metadomain_comm.cpp +++ b/src/framework/domain/metadomain_comm.cpp @@ -3,6 +3,7 @@ #include "arch/directions.h" #include "arch/kokkos_aliases.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/error.h" #include "utils/formatting.h" @@ -204,74 +205,66 @@ namespace ntt { template void Metadomain::CommunicateFields(Domain& domain, CommTags tags) const { - const auto comm_em = ((S == SimEngine::SRPIC) and - ((tags & Comm::E) or (tags & Comm::B))) or - ((S == SimEngine::GRPIC) and - ((tags & Comm::D) or (tags & Comm::B))); - const bool comm_em0 = (S == SimEngine::GRPIC) and - ((tags & Comm::B0) or (tags & Comm::D0)); - const bool comm_j = (tags & Comm::J); - const bool comm_aux = (S == SimEngine::GRPIC) and - ((tags & Comm::E) or (tags & Comm::H)); - raise::ErrorIf(not(comm_em or comm_em0 or comm_j or comm_aux), + const auto comm_em = (tags & Comm::EM_012) or (tags & Comm::EM_345); + const auto comm_em0 = (tags & Comm::EM0_012) or (tags & Comm::EM0_345); + const auto comm_cur = (tags & Comm::CUR); + const auto comm_cur0 = (tags & Comm::CUR0); + const auto comm_aux = (tags & Comm::AUX_012) or (tags & Comm::AUX_345); + const auto comm_bckp = (tags & Comm::Bckp); + + raise::ErrorIf(not(comm_em or comm_em0 or comm_cur or comm_aux or comm_cur0 or + comm_bckp), "CommunicateFields called with no task", HERE); + if constexpr (not ::traits::engine::DefinesEM0Fields) { + raise::ErrorIf(comm_em0, + "CommunicateFields called with EM0 communication " + "for an engine that does not define EM0 fields", + HERE); + } + if constexpr (not ::traits::engine::DefinesAuxFields) { + raise::ErrorIf(comm_aux, + "CommunicateFields called with AUX communication " + "for an engine that does not define AUX fields", + HERE); + } + if constexpr (not ::traits::engine::DefinesCur0Fields) { + raise::ErrorIf(comm_cur0, + "CommunicateFields called with CUR0 communication " + "for an engine that does not define CUR0 fields", + HERE); + } std::string comms; - if (tags & Comm::E) { - comms += "E "; + if (tags & Comm::EM_012) { + comms += "EM[0-2] "; } - if (tags & Comm::B) { - comms += "B "; + if (tags & Comm::EM_345) { + comms += "EM[3-5] "; } - if (tags & Comm::J) { - comms += "J "; + if (comm_cur) { + comms += "CUR "; } - if (tags & Comm::D) { - comms += "D "; + if (tags & Comm::AUX_012) { + comms += "AUX[0-2] "; } - if (tags & Comm::H) { - comms += "H "; + if (tags & Comm::AUX_345) { + comms += "AUX[3-5] "; } - if (tags & Comm::D0) { - comms += "D0 "; + if (tags & Comm::EM0_012) { + comms += "EM0[0-2] "; } - if (tags & Comm::B0) { - comms += "B0 "; + if (tags & Comm::EM0_345) { + comms += "EM0[3-5] "; } - logger::Checkpoint(fmt::format("Communicating %s\n", comms.c_str()), HERE); - - /** - * @note this block is designed to support in the future multiple domains - * on a single rank, however that is not yet implemented - */ - // establish the last index ranges for fields (i.e., components) - auto comp_range_fld = cell_range_t {}; - auto comp_range_cur = cell_range_t {}; - if constexpr (S == SimEngine::GRPIC) { - if (((tags & Comm::D) and (tags & Comm::B)) or - ((tags & Comm::D0) and (tags & Comm::B0)) or - ((tags & Comm::E) and (tags & Comm::H))) { - comp_range_fld = cell_range_t(em::dx1, em::bx3 + 1); - } else if ((tags & Comm::D) or (tags & Comm::D0) or (tags & Comm::E)) { - comp_range_fld = cell_range_t(em::dx1, em::dx3 + 1); - } else if ((tags & Comm::B) or (tags & Comm::B0) or (tags & Comm::H)) { - comp_range_fld = cell_range_t(em::bx1, em::bx3 + 1); - } - } else if constexpr (S == SimEngine::SRPIC) { - if ((tags & Comm::E) and (tags & Comm::B)) { - comp_range_fld = cell_range_t(em::ex1, em::bx3 + 1); - } else if (tags & Comm::E) { - comp_range_fld = cell_range_t(em::ex1, em::ex3 + 1); - } else if (tags & Comm::B) { - comp_range_fld = cell_range_t(em::bx1, em::bx3 + 1); - } - } else { - raise::Error("Unknown simulation engine", HERE); + if (comm_cur0) { + comms += "CUR0 "; } - if (comm_j) { - comp_range_cur = cell_range_t(cur::jx1, cur::jx3 + 1); + if (comm_bckp) { + comms += "Bckp "; } + logger::Checkpoint(fmt::format("Communicating %s\n", comms.c_str()), HERE); + // traverse in all directions and send/recv the fields for (auto& direction : dir::Directions::all) { const auto [send_params, @@ -284,6 +277,16 @@ namespace ntt { continue; } if (comm_em) { + auto comp_range = cell_range_t {}; + if ((tags & Comm::EM_012) and (tags & Comm::EM_345)) { + comp_range = cell_range_t(0, 6); + } else if (tags & Comm::EM_012) { + comp_range = cell_range_t(0, 3); + } else if (tags & Comm::EM_345) { + comp_range = cell_range_t(3, 6); + } else { + raise::Error("Incorrect logic", HERE); + } comm::CommunicateField(domain.index(), domain.fields.em, domain.fields.em, @@ -293,75 +296,95 @@ namespace ntt { recv_rank, send_slice, recv_slice, - comp_range_fld, + comp_range, false); } - if constexpr (S == SimEngine::GRPIC) { - if (comm_aux) { - comm::CommunicateField(domain.index(), - domain.fields.aux, - domain.fields.aux, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_fld, - false); - } - if (comm_em0) { - comm::CommunicateField(domain.index(), - domain.fields.em0, - domain.fields.em0, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_fld, - false); - // @HACK_GR_1.2.0 -- this has to be done carefully - // comm::CommunicateField(domain.index(), - // domain.fields.aux, - // domain.fields.aux, - // send_ind, - // recv_ind, - // send_rank, - // recv_rank, - // send_slice, - // recv_slice, - // comp_range_fld, - // false); - } - if (comm_j) { - comm::CommunicateField(domain.index(), - domain.fields.cur0, - domain.fields.cur0, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - false); + if (comm_aux) { + auto comp_range = cell_range_t {}; + if ((tags & Comm::AUX_012) and (tags & Comm::AUX_345)) { + comp_range = cell_range_t(0, 6); + } else if (tags & Comm::AUX_012) { + comp_range = cell_range_t(0, 3); + } else if (tags & Comm::AUX_345) { + comp_range = cell_range_t(3, 6); + } else { + raise::Error("Incorrect logic", HERE); } - } else { - if (comm_j) { - comm::CommunicateField(domain.index(), - domain.fields.cur, - domain.fields.cur, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - false); + comm::CommunicateField(domain.index(), + domain.fields.aux, + domain.fields.aux, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range, + false); + } + if (comm_em0) { + auto comp_range = cell_range_t {}; + if ((tags & Comm::EM0_012) and (tags & Comm::EM0_345)) { + comp_range = cell_range_t(0, 6); + } else if (tags & Comm::EM0_012) { + comp_range = cell_range_t(0, 3); + } else if (tags & Comm::EM0_345) { + comp_range = cell_range_t(3, 6); + } else { + raise::Error("Incorrect logic", HERE); } + comm::CommunicateField(domain.index(), + domain.fields.em0, + domain.fields.em0, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range, + false); + } + if (comm_cur0) { + comm::CommunicateField(domain.index(), + domain.fields.cur0, + domain.fields.cur0, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 3 }, + false); + } + if (comm_cur) { + auto comp_range = cell_range_t(0, 3); + comm::CommunicateField(domain.index(), + domain.fields.cur, + domain.fields.cur, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 3 }, + false); + } + if (comm_bckp) { + // copy active -> ghost of bckp (Ec/Bc); read by the hybrid pusher gather + comm::CommunicateField(domain.index(), + domain.fields.bckp, + domain.fields.bckp, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 6 }, + false); } } } @@ -409,20 +432,45 @@ namespace ntt { void Metadomain::SynchronizeFields(Domain& domain, CommTags tags, const cell_range_t& components) const { - const bool comm_j = (tags & Comm::J); + const bool comm_cur = (tags & Comm::CUR); + const bool comm_cur0 = (tags & Comm::CUR0); const bool comm_bckp = (tags & Comm::Bckp); const bool comm_buff = (tags & Comm::Buff); - raise::ErrorIf(not(comm_j || comm_bckp || comm_buff), - "SynchronizeFields called with no task or incorrect task", - HERE); - raise::ErrorIf(comm_j and comm_buff, - "SynchronizeFields cannot sync J and Buff at the same time", + const bool comm_aux = (tags & Comm::AUX_012) || (tags & Comm::AUX_345); + raise::ErrorIf( + not(comm_cur0 || comm_cur || comm_bckp || comm_buff || comm_aux), + "SynchronizeFields called with no task or incorrect task", + HERE); + raise::ErrorIf( + (comm_cur0 and comm_buff) or (comm_cur and comm_buff), + "SynchronizeFields cannot sync CUR/CUR0 and Buff at the same time", + HERE); + raise::ErrorIf((comm_cur0 and comm_cur), + "SynchronizeFields cannot sync CUR and CUR0 at the same " + "time (both use Buff as buffer)", HERE); - const auto synchronize = true; + + if constexpr (not ::traits::engine::DefinesCur0Fields) { + raise::ErrorIf(comm_cur0, + "CommunicateFields called with CUR0 communication " + "for an engine that does not define CUR0 fields", + HERE); + } + if constexpr (not ::traits::engine::DefinesAuxFields) { + raise::ErrorIf(comm_aux, + "SynchronizeFields called with AUX synchronization " + "for an engine that does not define AUX fields", + HERE); + } + + const auto SYNCHRONIZE = true; std::string comms; - if (comm_j) { - comms += "J "; + if (comm_cur) { + comms += "CUR "; + } + if (comm_cur0) { + comms += "CUR0 "; } if (comm_bckp) { comms += "Bckp "; @@ -430,14 +478,13 @@ namespace ntt { if (comm_buff) { comms += "Buff "; } + if (comm_aux) { + comms += "AUX "; + } logger::Checkpoint(fmt::format("Synchronizing %s\n", comms.c_str()), HERE); - auto comp_range_cur = cell_range_t {}; - if (comm_j) { - comp_range_cur = cell_range_t(cur::jx1, cur::jx3 + 1); - Kokkos::deep_copy(domain.fields.buff, ZERO); - } ndfield_t bckp_recv; + ndfield_t aux_recv; ndfield_t buff_recv; if (comm_bckp) { if constexpr (M::Dim == Dim::_1D) { @@ -454,6 +501,21 @@ namespace ntt { domain.fields.bckp.extent(2) }; } } + if (comm_aux) { + if constexpr (M::Dim == Dim::_1D) { + aux_recv = ndfield_t { "aux_recv", + domain.fields.aux.extent(0) }; + } else if constexpr (M::Dim == Dim::_2D) { + aux_recv = ndfield_t { "aux_recv", + domain.fields.aux.extent(0), + domain.fields.aux.extent(1) }; + } else if constexpr (M::Dim == Dim::_3D) { + aux_recv = ndfield_t { "aux_recv", + domain.fields.aux.extent(0), + domain.fields.aux.extent(1), + domain.fields.aux.extent(2) }; + } + } if (comm_buff) { if constexpr (M::Dim == Dim::_1D) { buff_recv = ndfield_t { "buff_recv", @@ -469,6 +531,13 @@ namespace ntt { domain.fields.buff.extent(2) }; } } + // buff accumulates the received deposit tails from every direction and is + // added into cur/cur0 once after the loop, so it must be zeroed exactly + // once, here, before the loop. Zeroing it per direction would keep only the + // last direction's contribution. + if (comm_cur or comm_cur0) { + Kokkos::deep_copy(domain.fields.buff, ZERO); + } // traverse in all directions and sync the fields for (auto& direction : dir::Directions::all) { const auto [send_params, @@ -480,32 +549,30 @@ namespace ntt { if (send_rank < 0 and recv_rank < 0) { continue; } - if (comm_j) { - if constexpr (S == SimEngine::GRPIC) { - comm::CommunicateField(domain.index(), - domain.fields.cur0, - domain.fields.buff, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - synchronize); - } else { - comm::CommunicateField(domain.index(), - domain.fields.cur, - domain.fields.buff, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - synchronize); - } + if (comm_cur) { + comm::CommunicateField(domain.index(), + domain.fields.cur, + domain.fields.buff, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 3 }, + SYNCHRONIZE); + } else if (comm_cur0) { + comm::CommunicateField(domain.index(), + domain.fields.cur0, + domain.fields.buff, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 3 }, + SYNCHRONIZE); } if (comm_bckp) { comm::CommunicateField(domain.index(), @@ -518,7 +585,22 @@ namespace ntt { send_slice, recv_slice, components, - synchronize); + SYNCHRONIZE); + } + if (comm_aux) { + // additive remap of moment deposit tails (Pegasus ยง3.6): accumulate the + // ghost-cell contributions of aux (V in 0..2, N in 3) into aux_recv + comm::CommunicateField(domain.index(), + domain.fields.aux, + aux_recv, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 6 }, + SYNCHRONIZE); } if (comm_buff) { comm::CommunicateField(domain.index(), @@ -531,21 +613,19 @@ namespace ntt { send_slice, recv_slice, components, - synchronize); + SYNCHRONIZE); } } - if (comm_j) { - if constexpr (S == SimEngine::GRPIC) { - AddBufferedFields(domain.fields.cur0, - domain.fields.buff, - domain.mesh.rangeActiveCells(), - comp_range_cur); - } else { - AddBufferedFields(domain.fields.cur, - domain.fields.buff, - domain.mesh.rangeActiveCells(), - comp_range_cur); - } + if (comm_cur) { + AddBufferedFields(domain.fields.cur, + domain.fields.buff, + domain.mesh.rangeActiveCells(), + { 0, 3 }); + } else if (comm_cur0) { + AddBufferedFields(domain.fields.cur0, + domain.fields.buff, + domain.mesh.rangeActiveCells(), + { 0, 3 }); } if (comm_bckp) { AddBufferedFields(domain.fields.bckp, @@ -553,6 +633,12 @@ namespace ntt { domain.mesh.rangeActiveCells(), components); } + if (comm_aux) { + AddBufferedFields(domain.fields.aux, + aux_recv, + domain.mesh.rangeActiveCells(), + { 0, 6 }); + } if (comm_buff) { AddBufferedFields(domain.fields.buff, buff_recv, diff --git a/src/framework/domain/metadomain_io.cpp b/src/framework/domain/metadomain_io.cpp index d53dc49a1..d242c57e1 100644 --- a/src/framework/domain/metadomain_io.cpp +++ b/src/framework/domain/metadomain_io.cpp @@ -548,7 +548,7 @@ namespace ntt { HERE); } } else if (fld.is_vpotential()) { - if constexpr (S == SimEngine::GRPIC && M::Dim == Dim::_2D) { + if constexpr (::traits::engine::IsGR && M::Dim == Dim::_2D) { const auto c = static_cast(addresses.back()); ComputeVectorPotential(local_domain->fields.bckp, local_domain->fields.em, @@ -611,7 +611,7 @@ namespace ntt { SynchronizeFields(*local_domain, Comm::Bckp, { addresses[0], addresses[2] + 1 }); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::IsSR) { if (fld.id() == FldsID::V) { // normalize 3vel * rho (combuted above) by rho ComputeMoments(params, @@ -646,7 +646,7 @@ namespace ntt { { cur::jx1, cur::jx3 + 1 }, copy_to); } else if (fld.is_field()) { - if (S == SimEngine::GRPIC && fld.is_gr_aux_field()) { + if (::traits::engine::IsGR && fld.is_gr_aux_field()) { if (fld.is_efield()) { // GR: E DeepCopyFields(local_domain->fields.aux, @@ -699,7 +699,7 @@ namespace ntt { } } } else if (fld.comp.size() == 4) { // 4-vector - if constexpr (S == SimEngine::GRPIC) { + if constexpr (::traits::engine::IsGR) { if (fld.is_moment() && fld.id() == FldsID::V) { // Compute 4-velocity: V^ฮผ (u^0, u^1, u^2, u^3) for (auto i = 0; i < 4; ++i) { diff --git a/src/framework/domain/metadomain_stats.cpp b/src/framework/domain/metadomain_stats.cpp index 35ff26e75..76635fc61 100644 --- a/src/framework/domain/metadomain_stats.cpp +++ b/src/framework/domain/metadomain_stats.cpp @@ -1,6 +1,7 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/comparators.h" #include "utils/error.h" @@ -137,7 +138,7 @@ namespace ntt { raise::Error("Components not supported for JdotE", HERE); } } else if constexpr ( - (S == SimEngine::SRPIC) and + (::traits::engine::IsSR) and (F == StatsID::B2 or F == StatsID::E2 or F == StatsID::ExB)) { raise::ErrorIf(components.size() != 1, "Components must be of size 1 for B2, E2 or ExB stats", @@ -251,7 +252,7 @@ namespace ntt { } else if (stat.id() == StatsID::JdotE) { g_stats_writer.write( ReduceFields(local_domain, g_mesh.metric, {})); - } else if (S == SimEngine::SRPIC) { + } else if (::traits::engine::IsSR) { if (stat.id() == StatsID::E2) { for (const auto& comp : stat.comp) { g_stats_writer.write( diff --git a/src/framework/parameters/grid.cpp b/src/framework/parameters/grid.cpp index b701ab98e..84088f367 100644 --- a/src/framework/parameters/grid.cpp +++ b/src/framework/parameters/grid.cpp @@ -4,6 +4,7 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" #include "utils/error.h" #include "utils/formatting.h" #include "utils/numeric.h" @@ -122,10 +123,10 @@ namespace ntt { std::vector> flds_bc_enum; std::vector> prtl_bc_enum; if (coord_enum == Coord::Cartesian) { - raise::ErrorIf(flds_bc.size() != (std::size_t)dim, + raise::ErrorIf(flds_bc.size() != (size_t)dim, "invalid `grid.boundaries.fields`", HERE); - raise::ErrorIf(prtl_bc.size() != (std::size_t)dim, + raise::ErrorIf(prtl_bc.size() != (size_t)dim, "invalid `grid.boundaries.particles`", HERE); for (auto d { 0u }; d < (dim_t)dim; ++d) { @@ -181,7 +182,7 @@ namespace ntt { raise::ErrorIf(prtl_bc.size() > 1, "invalid `grid.boundaries.particles`", HERE); - if (engine_enum == SimEngine::SRPIC) { + if (::traits::engine::isSR(engine_enum)) { raise::ErrorIf(flds_bc[0].size() != 2, "invalid `grid.boundaries.fields`", HERE); @@ -224,10 +225,10 @@ namespace ntt { } } - raise::ErrorIf(flds_bc_enum.size() != (std::size_t)dim, + raise::ErrorIf(flds_bc_enum.size() != (size_t)dim, "invalid inferred `grid.boundaries.fields`", HERE); - raise::ErrorIf(prtl_bc_enum.size() != (std::size_t)dim, + raise::ErrorIf(prtl_bc_enum.size() != (size_t)dim, "invalid inferred `grid.boundaries.particles`", HERE); boundaries_t flds_bc_pairwise; @@ -421,8 +422,8 @@ namespace ntt { metric_params_short_.emplace(); std::string coord; if (metric_enum == Metric::Minkowski) { - raise::ErrorIf(engine_enum != SimEngine::SRPIC, - "minkowski metric is only supported for SRPIC", + raise::ErrorIf(not ::traits::engine::isSR(engine_enum), + "minkowski metric is only supported for SR", HERE); coord = "cart"; } else if (metric_enum == Metric::QKerr_Schild or @@ -455,7 +456,7 @@ namespace ntt { HERE); coord = "sph"; } - if ((engine_enum == SimEngine::GRPIC) && + if (::traits::engine::isGR(engine_enum) && (metric_enum != Metric::Kerr_Schild_0)) { const auto ks_a = toml::find_or(toml_data, "grid", @@ -501,7 +502,7 @@ namespace ntt { (*metric_params_short_)["r0"] = (*metric_params)["qsph_r0"]; (*metric_params_short_)["h"] = (*metric_params)["qsph_h"]; } - if ((engine_enum == SimEngine::GRPIC) && + if (::traits::engine::isGR(engine_enum) && (metric_enum != Metric::Kerr_Schild_0)) { (*metric_params_short_)["a"] = (*metric_params)["ks_a"]; } diff --git a/src/framework/parameters/parameters.cpp b/src/framework/parameters/parameters.cpp index 8b525d49a..4b35f7e67 100644 --- a/src/framework/parameters/parameters.cpp +++ b/src/framework/parameters/parameters.cpp @@ -54,6 +54,87 @@ namespace ntt { set("scales.sigma0", SQR(skindepth0 / larmor0)); set("scales.B0", ONE / larmor0); set("scales.omegaB0", ONE / larmor0); + if (engine_enum == SimEngine::HYBRID) { + /** + * hybrid-engine parameters (read from the [hybrid] table; ignored by the + * SR/GR engines): + * gamma_ad - adiabatic index of the massless electron fluid (1 = isothermal) + * theta0 - electron temperature T_e (code units); 0 = cold electrons + * dens_min - vacuum threshold for the Ohm's law; below it E ramps to 0 (units of n0) + * hall_lim - cap on the local whistler Courant the Hall term may imply (<= 0 off) + * resist_vac - vacuum resistivity: E -> eta curl B below dens_min (0 = off) + * resist_hyper - hyper-resistivity eta_H grad^4 B, everywhere (0 = off) + * subcycle - Pegasus-style sub-cycled field advance (default on) + * subcycle_courant - target whistler Courant per field sub-step + * subcycle_max - cap on the number of field sub-steps per advance + * v_max - characteristic flow speed for the hybrid CFL (code units) + */ + set("hybrid.gamma_ad", + toml::find_or(toml_data, + "hybrid", + "gamma_ad", + defaults::hybrid::gamma_ad)); + set("hybrid.theta0", + toml::find_or(toml_data, "hybrid", "theta0", ZERO)); + // vacuum threshold for the hybrid Ohm's law. Above it E follows the usual + // [...]/N; below it E ramps continuously to zero so that plasma-free cells + // keep B frozen instead of amplifying the right-hand side by 1/dens_min. + set("hybrid.dens_min", + toml::find_or(toml_data, + "hybrid", + "dens_min", + defaults::hybrid::dens_min)); + // per-cell limiter on the Ohm's-law Hall term: caps the implied local + // whistler Courant (v_w ~ |B|/N, which exceeds the ambient value at shock + // overshoots and magnetic cavities) at hall_lim * dx/dt. Identity in + // resolvable cells; <= 0 disables. + set("hybrid.hall_lim", + toml::find_or(toml_data, + "hybrid", + "hall_lim", + static_cast(0.5))); + // vacuum resistivity (code units). > 0 turns the E = 0 vacuum of dens_min + // into a resistive vacuum: below the threshold compute_Ee crossfades E to + // eta * curl B, so vacuum regions diffuse toward a curl-free (potential) + // field instead of freezing. Needed when fast plasma sweeps along a vacuum + // boundary, where a hard E = 0 mask makes an artificial tangential-E jump + // (a surface current) that pumps B at rate ~ u/dx. Internally clamped to + // the explicit-diffusion stability limit (0.4 dx^2/dt at the sub-step dt), + // so any value is safe; eta ~ u_sweep * dx suffices. + set("hybrid.resist_vac", + toml::find_or(toml_data, "hybrid", "resist_vac", ZERO)); + // hyper-resistivity (E -= eta_H grad^2 curl B => dB/dt -= eta_H grad^4 B), + // applied everywhere in the Faraday-side Ohm's law. The moments filter + // (algorithms.current_filters) attenuates the deposited N, V by + // cos^(2p)(k dx/2) -- near-zero for few-cell wavelengths -- so in that band + // the field evolves with no kinetic feedback and sustained compression or + // shear in the frozen moments pumps the band by induction. eta_H k^4 + // blankets the filtered band while leaving k dx < 0.3 untouched; size so + // eta_H (pi/dx)^4 exceeds the drive. Internally clamped to the explicit + // grad^4 stability limit at the actual sub-step dt. + set("hybrid.resist_hyper", + toml::find_or(toml_data, "hybrid", "resist_hyper", ZERO)); + // Pegasus-style sub-cycled magnetic-field advance (fieldsolvers.h + // ::SubcycledFaraday): the whistler-stiff Ohm/Faraday loop is integrated + // with adaptive SSP-RK3 sub-steps at its own CFL instead of full-dt Euler + // pushes. The sub-step count targets `subcycle_courant` whistler Courant + // per sub-step (from the global max |B|/N) and is capped at `subcycle_max`; + // beyond the cap the per-cell Hall limiter (which then sees the SUB-step + // dt) takes over. subcycle = false -> legacy 3-push scheme. + set("hybrid.subcycle", + toml::find_or(toml_data, "hybrid", "subcycle", true)); + set("hybrid.subcycle_courant", + toml::find_or(toml_data, + "hybrid", + "subcycle_courant", + static_cast(0.5))); + set("hybrid.subcycle_max", + toml::find_or(toml_data, "hybrid", "subcycle_max", 64)); + // optional user-set characteristic flow speed for the hybrid CFL (code + // units); 0 -> dt set purely by the Alfven + whistler signal speeds. + set("hybrid.v_max", + toml::find_or(toml_data, "hybrid", "v_max", ZERO)); + } /* [particles] ---------------------------------------------------------- */ const auto ppc0 = toml::find(toml_data, "particles", "ppc0"); @@ -239,6 +320,38 @@ namespace ntt { alg_params.read(get("scales.dx0"), alg_extra_flags, toml_data); alg_params.setParams(alg_extra_flags, this); + /* hybrid timestep โ€” correct CFL --------------------------------------- */ + // The default light-crossing dt = CFL * dx0 assumes the fastest signal + // travels at ~1 code-velocity unit (as for SR/GR, where speeds are bounded + // by c = 1). A hybrid plasma has no displacement current; its fastest + // signals are the DISPERSIVE whistler/Hall mode and the bulk ion flow, both + // larger than 1 in code units. Using dt = CFL * dx0 then advects particles + // by > 1 cell/step (Courant > 1), which breaks the single-cell rebucket and + // the moment deposit. Replace it with the Pegasus CFL (Kunz, Stone & Bai + // 2014, ยง3.5): dt <= CFL * min( dx0 / v_max, 2*pi/Omega ), with the + // ground-frame max signal speed + // v_max = v_flow + v_A + v_whistler, + // v_A = d0 / rho0 (Alfven speed, code units), + // v_whistler = (d0^2 / rho0) * pi / dx0 (grid-scale Hall/whistler; + // d0^2/rho0 is the EMF Hall coefficient coeff_2, k_max~pi/dx0), + // and v_flow an optional user-set characteristic flow speed in code units + // (hybrid.v_max, default 0 โ€” set it for super-whistler flows). + if (engine_enum == SimEngine::HYBRID) { + const auto cfl = get("algorithms.timestep.CFL"); + const auto dx0 = get("scales.dx0"); + const auto d0 = get("scales.skindepth0"); + const auto rho0 = get("scales.larmor0"); + const auto v_flow = get("hybrid.v_max"); + const auto v_alfven = d0 / rho0; + const auto v_whistler = (SQR(d0) / rho0) * + static_cast(constant::PI) / dx0; + const auto v_max = v_flow + v_alfven + v_whistler; + const auto dt_adv = dx0 / v_max; + // gyration limit 2*pi/Omega (Omega = 1/rho0); rarely binding + const auto dt_gyr = static_cast(constant::TWO_PI) * rho0; + const auto dt_hyb = cfl * ((dt_adv < dt_gyr) ? dt_adv : dt_gyr); + set("algorithms.timestep.dt", dt_hyb); + } /* [simulation.domain.load_balance] ------------------------------------- */ set("simulation.domain.load_balance.enable", toml::find_or(toml_data, "simulation", "domain", "load_balance", "enable", false)); diff --git a/src/framework/specialization_registry.h b/src/framework/specialization_registry.h index dea5772c6..d276d4a84 100644 --- a/src/framework/specialization_registry.h +++ b/src/framework/specialization_registry.h @@ -37,6 +37,16 @@ namespace ntt { static constexpr auto dimension = D; }; +#define NTT_FOREACH_SPECIALIZATION_FIELDS(MACRO) \ + MACRO(Dim::_1D, SimEngine::SRPIC) \ + MACRO(Dim::_2D, SimEngine::SRPIC) \ + MACRO(Dim::_3D, SimEngine::SRPIC) \ + MACRO(Dim::_2D, SimEngine::GRPIC) \ + MACRO(Dim::_3D, SimEngine::GRPIC) \ + MACRO(Dim::_1D, SimEngine::HYBRID) \ + MACRO(Dim::_2D, SimEngine::HYBRID) \ + MACRO(Dim::_3D, SimEngine::HYBRID) + #define NTT_FOREACH_SPECIALIZATION(MACRO) \ MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_1D) \ MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_2D) \ @@ -45,12 +55,18 @@ namespace ntt { MACRO(SimEngine::SRPIC, metric::QSpherical, Dim::_2D) \ MACRO(SimEngine::GRPIC, metric::KerrSchild, Dim::_2D) \ MACRO(SimEngine::GRPIC, metric::QKerrSchild, Dim::_2D) \ - MACRO(SimEngine::GRPIC, metric::KerrSchild0, Dim::_2D) + MACRO(SimEngine::GRPIC, metric::KerrSchild0, Dim::_2D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_1D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_2D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_3D) #define NTT_FOREACH_CARTESIAN_SPECIALIZATION(MACRO) \ MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_1D) \ MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_2D) \ - MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_3D) + MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_3D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_1D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_2D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_3D) #define NTT_BUILD_SPECIALIZATION_ENTRY(S, M, D) SpecializationEntry {}, diff --git a/src/global/defaults.h b/src/global/defaults.h index c4f9b67f4..9b0074ab8 100644 --- a/src/global/defaults.h +++ b/src/global/defaults.h @@ -42,6 +42,11 @@ namespace ntt::defaults { const real_t beta_zy = 0.0; } // namespace fieldsolver + namespace hybrid { + const real_t gamma_ad = static_cast(5.0 / 3.0); + const real_t dens_min = static_cast(1e-3); + } // namespace hybrid + namespace qsph { const real_t r0 = 0.0; const real_t h = 0.0; diff --git a/src/global/enums.h b/src/global/enums.h index 5700a3a28..6136bd653 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -5,7 +5,7 @@ * - enum ntt::Coord // Cart, Sph, Qsph * - enum ntt::Metric // Minkowski, Spherical, QSpherical, * Kerr_Schild, QKerr_Schild, Kerr_Schild_0 - * - enum ntt::SimEngine // SRPIC, GRPIC + * - enum ntt::SimEngine // SRPIC, GRPIC, HYBRID * - enum ntt::PrtlBC // periodic, absorb, atmosphere, custom, * reflect, horizon, axis, sync * - enum ntt::FldsBC // periodic, match, fixed, atmosphere, @@ -59,8 +59,8 @@ namespace ntt { namespace enums_hidden { template class EnumBase { - constexpr auto idx() const -> std::size_t { - return static_cast(d().val) - 1; // assumes 1-indexed + constexpr auto idx() const -> size_t { + return static_cast(d().val) - 1; // assumes 1-indexed } constexpr auto d() const -> const Derived& { @@ -131,7 +131,7 @@ namespace ntt { static constexpr const char* label = "coord"; static constexpr type variants[] = { Cartesian, Spherical, Qspherical }; static constexpr const char* lookup[] = { "cart", "sph", "qsph" }; - static constexpr std::size_t total = std::size(variants); + static constexpr size_t total = std::size(variants); }; struct Metric : public enums_hidden::EnumBase { @@ -159,7 +159,7 @@ namespace ntt { static constexpr const char* lookup[] = { "minkowski", "spherical", "qspherical", "kerr_schild", "qkerr_schild", "kerr_schild_0" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; struct SimEngine : public enums_hidden::EnumBase { @@ -167,6 +167,7 @@ namespace ntt { INVALID = 0, SRPIC = 1, GRPIC = 2, + HYBRID = 3, }; using enum type; type val; @@ -177,9 +178,9 @@ namespace ntt { : val { v } {} static constexpr const char* label = "sim_engine"; - static constexpr type variants[] = { SRPIC, GRPIC }; - static constexpr const char* lookup[] = { "srpic", "grpic" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr type variants[] = { SRPIC, GRPIC, HYBRID }; + static constexpr const char* lookup[] = { "srpic", "grpic", "hybrid" }; + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; struct PrtlBC : public enums_hidden::EnumBase { @@ -209,7 +210,7 @@ namespace ntt { "atmosphere", "custom", "reflect", "horizon", "axis", "sync" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; struct FldsBC : public enums_hidden::EnumBase { @@ -242,7 +243,7 @@ namespace ntt { "periodic", "match", "fixed", "atmosphere", "custom", "horizon", "axis", "conductor", "sync" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; struct FldsID : public enums_hidden::EnumBase { @@ -280,7 +281,7 @@ namespace ntt { "b", "h", "j", "a", "t", "rho", "charge", "n", "nppc", "v", "custom" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; struct StatsID : public enums_hidden::EnumBase { @@ -311,7 +312,7 @@ namespace ntt { static constexpr const char* lookup[] = { "b^2", "e^2", "exb", "j.e", "t", "rho", "charge", "n", "npart", "custom" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; namespace ParticlePusher { diff --git a/src/global/global.h b/src/global/global.h index c3fbc5061..335be097f 100644 --- a/src/global/global.h +++ b/src/global/global.h @@ -154,6 +154,15 @@ namespace ntt { hx3 = 5 }; + enum comp : uint8_t { + c0 = 0, + c1 = 1, + c2 = 2, + c3 = 3, + c4 = 4, + c5 = 5, + }; + enum cur : uint8_t { jx1 = 0, jx2 = 1, @@ -276,16 +285,20 @@ using DiagFlags = uint8_t; namespace Comm { enum CommTags_ : uint16_t { - None = 0, - E = 1 << 0, - B = 1 << 1, - J = 1 << 2, - D = 1 << 3, - D0 = 1 << 4, - B0 = 1 << 5, - H = 1 << 6, - Bckp = 1 << 7, - Buff = 1 << 8, + None = 0, + EM_012 = 1 << 0, + EM_345 = 1 << 1, + CUR = 1 << 2, + AUX_012 = 1 << 3, + AUX_345 = 1 << 4, + EM0_012 = 1 << 5, + EM0_345 = 1 << 6, + CUR0 = 1 << 7, + Bckp = 1 << 8, + Buff = 1 << 9, + EM = EM_012 | EM_345, + AUX = AUX_012 | AUX_345, + EM0 = EM0_012 | EM0_345, }; } // namespace Comm diff --git a/src/global/traits/archetypes.h b/src/global/traits/archetypes.h index e170170fb..4c9c443c1 100644 --- a/src/global/traits/archetypes.h +++ b/src/global/traits/archetypes.h @@ -24,6 +24,8 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" + #include template @@ -202,11 +204,11 @@ namespace traits::fieldsetter { template concept FieldSetterClass = - ((S == ntt::SimEngine::SRPIC) and + ((::traits::engine::IsSR) and (::traits::fieldsetter::HasEx1 or ::traits::fieldsetter::HasEx2 or ::traits::fieldsetter::HasEx3 or ::traits::fieldsetter::HasBx1 or ::traits::fieldsetter::HasBx2 or ::traits::fieldsetter::HasBx3)) or - ((S == ntt::SimEngine::GRPIC) and + ((::traits::engine::IsGR) and ((::traits::fieldsetter::HasDx1 and ::traits::fieldsetter::HasDx2 and ::traits::fieldsetter::HasDx3) or (::traits::fieldsetter::HasBx1 and ::traits::fieldsetter::HasBx2 and diff --git a/src/global/traits/engine.h b/src/global/traits/engine.h new file mode 100644 index 000000000..1993c828d --- /dev/null +++ b/src/global/traits/engine.h @@ -0,0 +1,95 @@ +/** + * @file traits/engine.h + * @brief Defines a set of traits to check if an engine class satisfies certain conditions + * @implements + * - IsSR<> - checks if engine is a special relativistic engine + * - isSR() - checks if a SimEngine enum instance corresponds to a special relativistic engine + * - IsGR<> - checks if engine is a general relativistic engine + * - isGR() - checks if a SimEngine enum instance corresponds to a general relativistic engine + * - VelocitiesInCartesianBasis<> - checks if engine has velocities in Cartesian basis + * - VelocitiesInCovariantBasis<> - checks if engine has velocities in covariant basis + * - StressEnergyInTetradBasis<> - checks if engine needs stress-energy tensor in tetrad basis + * - StressEnergyInContravariantBasis<> - checks if engine needs stress-energy tensor in contravariant basis + * - UserFieldsInTetradBasis<> - checks if engine expects user-defined fields in tetrad basis + * - userFieldsInTetradBasis() - checks if a SimEngine enum instance corresponds + * to an engine that expects user-defined fields in tetrad basis + * - HasImplicitPhiCoordinate<> - checks if engine has particles with an implicit phi coordinate + * @namespaces: + * - ::traits::engine:: + */ +#ifndef TRAITS_ENGINE_H +#define TRAITS_ENGINE_H + +#include "enums.h" + +#include "traits/metric.h" + +namespace traits::engine { + + template + concept IsSR = (S == ntt::SimEngine::SRPIC) or (S == ntt::SimEngine::HYBRID); + + [[nodiscard]] + constexpr auto isSR(ntt::SimEngine s) noexcept -> bool { + return (s == ntt::SimEngine::SRPIC) or (s == ntt::SimEngine::HYBRID); + } + + template + concept IsGR = (S == ntt::SimEngine::GRPIC); + + [[nodiscard]] + constexpr auto isGR(ntt::SimEngine s) noexcept -> bool { + return s == ntt::SimEngine::GRPIC; + } + + template + concept VelocitiesInCartesianBasis = (S == ntt::SimEngine::SRPIC) or + (S == ntt::SimEngine::HYBRID); + + template + concept VelocitiesInCovariantBasis = (S == ntt::SimEngine::GRPIC); + + template + concept StressEnergyInTetradBasis = (S == ntt::SimEngine::SRPIC) or + (S == ntt::SimEngine::HYBRID); + + template + concept StressEnergyInContravariantBasis = (S == ntt::SimEngine::GRPIC); + + template + concept DefinesEM0Fields = (S == ntt::SimEngine::GRPIC) or + (S == ntt::SimEngine::HYBRID); + + template + concept DefinesAuxFields = (S == ntt::SimEngine::GRPIC) or + (S == ntt::SimEngine::HYBRID); + + template + concept DefinesCur0Fields = (S == ntt::SimEngine::GRPIC); + + template + concept WriteEM0FieldToCheckpoint = (S == ntt::SimEngine::GRPIC); + + template + concept WriteCurFieldToCheckpoint = (S == ntt::SimEngine::GRPIC); + + template + concept UserFieldsInTetradBasis = (S == ntt::SimEngine::SRPIC) or + (S == ntt::SimEngine::HYBRID); + + template + concept UserFieldsInContravariantBasis = (S == ntt::SimEngine::GRPIC); + + [[nodiscard]] + constexpr auto userFieldsInTetradBasis(ntt::SimEngine s) noexcept -> bool { + return (s == ntt::SimEngine::SRPIC) or (s == ntt::SimEngine::HYBRID); + } + + template + concept HasImplicitPhiCoordinate = IsSR and ::traits::metric::HasPrtlDim and + ::traits::metric::HasD and + M::Dim == Dim::_2D and M::PrtlDim == Dim::_3D; + +} // namespace traits::engine + +#endif // TRAITS_ENGINE_H diff --git a/src/kernels/currents_deposit.hpp b/src/kernels/currents_deposit.hpp index 98fa0b661..3050dc51d 100644 --- a/src/kernels/currents_deposit.hpp +++ b/src/kernels/currents_deposit.hpp @@ -28,6 +28,7 @@ #include "global.h" #include "arch/kokkos_aliases.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/error.h" #include "utils/numeric.h" diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 729a09d10..61e07f5c7 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -21,6 +21,7 @@ #include "arch/kokkos_aliases.h" #include "traits/archetypes.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/error.h" #include "utils/numeric.h" @@ -94,7 +95,7 @@ namespace kernel::bc { if constexpr (M::Dim == Dim::_1D) { const auto i1_ = COORD(i1); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { coord_t x_Ph_0 { ZERO }; coord_t x_Ph_H { ZERO }; metric.template convert({ i1_ }, x_Ph_0); @@ -163,8 +164,7 @@ namespace kernel::bc { } } } else { - // GRPIC - raise::KernelError(HERE, "1D GRPIC not implemented"); + raise::KernelError(HERE, "not implemented"); } } else { raise::KernelError( @@ -196,13 +196,15 @@ namespace kernel::bc { if constexpr (HasEx1 or HasDx1) { if ((tags & BC::E) or (tags & BC::D)) { - if constexpr (HasEx1 and S == SimEngine::SRPIC) { + if constexpr (HasEx1 and + ::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::ex1) = s * Fld(i1, i2, em::ex1) + (ONE - s) * metric.template transform<1, Idx::T, Idx::U>( { i1_ + HALF, i2_ }, fset.ex1(x_Ph_H0)); - } else if constexpr (HasDx1 and S == SimEngine::GRPIC) { + } else if constexpr ( + HasDx1 and (not ::traits::engine::UserFieldsInTetradBasis)) { Fld(i1, i2, em::dx1) = s * Fld(i1, i2, em::dx1) + (ONE - s) * fset.dx1(x_Ph_H0); } @@ -211,13 +213,13 @@ namespace kernel::bc { if constexpr (HasBx2) { if (tags & BC::B) { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::bx2) = s * Fld(i1, i2, em::bx2) + (ONE - s) * metric.template transform<2, Idx::T, Idx::U>( { i1_ + HALF, i2_ }, fset.bx2(x_Ph_H0)); - } else if constexpr (S == SimEngine::GRPIC) { + } else { Fld(i1, i2, em::bx2) = s * Fld(i1, i2, em::bx2) + (ONE - s) * fset.bx2(x_Ph_H0); } @@ -242,13 +244,15 @@ namespace kernel::bc { if constexpr (HasEx2 or HasDx2) { if ((tags & BC::E) or (tags & BC::D)) { - if constexpr (HasEx2 and S == SimEngine::SRPIC) { + if constexpr (HasEx2 and + ::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::ex2) = s * Fld(i1, i2, em::ex2) + (ONE - s) * metric.template transform<2, Idx::T, Idx::U>( { i1_, i2_ + HALF }, fset.ex2(x_Ph_0H)); - } else if constexpr (HasDx2 and S == SimEngine::GRPIC) { + } else if constexpr ( + HasDx2 and (not ::traits::engine::UserFieldsInTetradBasis)) { Fld(i1, i2, em::dx2) = s * Fld(i1, i2, em::dx2) + (ONE - s) * fset.dx2(x_Ph_0H); } @@ -257,13 +261,13 @@ namespace kernel::bc { if constexpr (HasBx1) { if (tags & BC::B) { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::bx1) = s * Fld(i1, i2, em::bx1) + (ONE - s) * metric.template transform<1, Idx::T, Idx::U>( { i1_, i2_ + HALF }, fset.bx1(x_Ph_0H)); - } else if constexpr (S == SimEngine::GRPIC) { + } else { Fld(i1, i2, em::bx1) = s * Fld(i1, i2, em::bx1) + (ONE - s) * fset.bx1(x_Ph_0H); } @@ -288,7 +292,7 @@ namespace kernel::bc { coord_t x_Ph_00 { ZERO }; metric.template convert({ i1_, i2_ }, x_Ph_00); - if constexpr (HasEx3 and S == SimEngine::SRPIC) { + if constexpr (HasEx3 and ::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::ex3) = s * Fld(i1, i2, em::ex3); if ((!is_axis_i2min or (i2 > N_GHOSTS)) and (!is_axis_i2max or (i2 < extent_2 - N_GHOSTS))) { @@ -297,7 +301,8 @@ namespace kernel::bc { { i1_, i2_ }, fset.ex3(x_Ph_00)); } - } else if constexpr (HasDx3 and S == SimEngine::GRPIC) { + } else if constexpr ( + HasDx3 and (not ::traits::engine::UserFieldsInTetradBasis)) { Fld(i1, i2, em::dx3) = s * Fld(i1, i2, em::dx3); if ((!is_axis_i2min or (i2 > N_GHOSTS)) and (!is_axis_i2max or (i2 < extent_2 - N_GHOSTS))) { @@ -324,13 +329,13 @@ namespace kernel::bc { metric.template convert({ i1_ + HALF, i2_ + HALF }, x_Ph_HH); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::bx3) = s * Fld(i1, i2, em::bx3) + (ONE - s) * metric.template transform<3, Idx::T, Idx::U>( { i1_ + HALF, i2_ + HALF }, fset.bx3(x_Ph_HH)); - } else if constexpr (S == SimEngine::GRPIC) { + } else { Fld(i1, i2, em::bx3) = s * Fld(i1, i2, em::bx3) + (ONE - s) * fset.bx3(x_Ph_HH); } @@ -349,7 +354,7 @@ namespace kernel::bc { const auto i2_ = COORD(i2); const auto i3_ = COORD(i3); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { // SRPIC if constexpr (HasEx1 or HasEx2 or HasEx3) { if (tags & BC::E) { @@ -509,8 +514,7 @@ namespace kernel::bc { } } } else { - // GRPIC - raise::KernelError(HERE, "GRPIC not implemented"); + raise::KernelError(HERE, "not implemented"); } } else { raise::KernelError( diff --git a/src/kernels/hybrid/EMF.hpp b/src/kernels/hybrid/EMF.hpp new file mode 100644 index 000000000..1e2348b1b --- /dev/null +++ b/src/kernels/hybrid/EMF.hpp @@ -0,0 +1,1088 @@ +#ifndef KERNELS_HYBRID_EMF_HPP +#define KERNELS_HYBRID_EMF_HPP + +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/numeric.h" + +#include + +namespace kernel::hybrid { + + /** + * Fields are stored in the contravariant (Idx::U) code basis of the Minkowski + * metric (cell size dx): components with index i <= D are physical/dx + * (h_ii = dx^2), out-of-plane components are physical (h_ii = 1). The + * deposited moments (PP = V, NN = N in `aux`) are physical (Cartesian XYZ + * velocity moments; N in units of n0). + * + * Inputs (Bf, Bfs, Ee_in) are U-basis. The outputs follow two conventions: + * - Ee_out (edge E, consumed by the Faraday curl and the trapezoidal + * average with Ee_in): U-basis, matching em/em0 storage. + * - Ec_out, Bc_out (cell-centered, read raw by the particle pusher and + * averaged with Ec = em0::012): physical (XYZ). + * + * Index-space differences of U-basis fields obey + * delta(b_inplane) = d(B_phys), delta(b_outofplane) = dx * d(B_phys), + * which is the origin of the per-term dx / dx_inv factors below. + */ + // EEONLY (requires INIT): compute and write ONLY the raw edge-E (Ee_out) -- + // used by the sub-cycled field advance, which refreshes Ee from frozen + // moments every sub-step and needs neither Ec nor Bc there. In this mode + // Bfs is the 3-component scratch (cur = the sub-stepped B) and Bf is unused + // (pass cur). + template + class EMF_kernel { + static_assert(INIT || not EEONLY, "EEONLY requires INIT (raw writes)"); + static constexpr uint8_t N1 = (INIT || EEONLY) ? 3 : 6; + static constexpr uint8_t N2 = (INIT && not EEONLY) ? 6 : 3; + + ndfield_t PP; + ndfield_t NN; + ndfield_t Ee_in; + ndfield_t Bf; + ndfield_t Ec; + ndfield_t Bfs; + + ndfield_t Ee_out; + ndfield_t Ec_out; + ndfield_t Bc_out; + + const uint8_t comp_PP; + const uint8_t comp_NN; + const uint8_t comp_Ee_in; + const uint8_t comp_Bf; + const uint8_t comp_Ec; + const uint8_t comp_Bfs; + + const uint8_t comp_Ee_out; + const uint8_t comp_Ec_out; + const uint8_t comp_Bc_out; + + const real_t dt; + const real_t gamma_ad; + const real_t theta; + const real_t d0; + const real_t rho0; + const real_t dens_min; + const real_t hall_lim; + const real_t resist_vac; + const real_t resist_hyper; + const real_t dx; + const real_t dx_inv; + + public: + EMF_kernel(const ndfield_t& PP, + const ndfield_t& NN, + const ndfield_t& Ee_in, + const ndfield_t& Bf, + const ndfield_t& Ec, + const ndfield_t& Bfs, + ndfield_t& Ee_out, + ndfield_t& Ec_out, + ndfield_t& Bc_out, + uint8_t comp_PP, + uint8_t comp_NN, + uint8_t comp_Ee_in, + uint8_t comp_Bf, + uint8_t comp_Ec, + uint8_t comp_Bfs, + uint8_t comp_Ee_out, + uint8_t comp_Ec_out, + uint8_t comp_Bc_out, + real_t dt, + real_t gamma_ad, + real_t theta, + real_t d0, + real_t rho0, + real_t dens_min, + real_t hall_lim, + real_t resist_vac, + real_t resist_hyper, + real_t dx) + : PP { PP } + , NN { NN } + , Ee_in { Ee_in } + , Bf { Bf } + , Ec { Ec } + , Bfs { Bfs } + , Ee_out { Ee_out } + , Ec_out { Ec_out } + , Bc_out { Bc_out } + , comp_PP { comp_PP } + , comp_NN { comp_NN } + , comp_Ee_in { comp_Ee_in } + , comp_Bf { comp_Bf } + , comp_Ec { comp_Ec } + , comp_Bfs { comp_Bfs } + , comp_Ee_out { comp_Ee_out } + , comp_Ec_out { comp_Ec_out } + , comp_Bc_out { comp_Bc_out } + , dt { dt } + , gamma_ad { gamma_ad } + , theta { theta } + , d0 { d0 } + , rho0 { rho0 } + , dens_min { dens_min } + , hall_lim { hall_lim } + , resist_vac { resist_vac } + , resist_hyper { resist_hyper } + , dx { dx } + , dx_inv { ONE / dx } {} + + /** + * @brief Effective vacuum resistivity, clamped to the explicit-diffusion + * stability limit eta dt / dx^2 <= 0.4 for this kernel's dt (the + * sub-step dt inside the sub-cycled advance), so the user knob can + * never destabilize the update. + */ + Inline auto eta_vac() const -> real_t { + return math::min(resist_vac, static_cast(0.4) * SQR(dx) / dt); + } + + /** + * @brief Effective HYPER-resistivity (E -= eta_H grad^2 curl B, i.e. + * dB/dt -= eta_H grad^4 B), applied everywhere in compute_Ee. + * + * The moments filter creates a spectral gap: p binomial passes attenuate + * the deposited N, V by cos^(2p)(k dx / 2), which is ~zero at wavelengths + * of a few cells. In that band the field solver evolves B with no kinetic + * feedback, so the ion damping that would physically kill those modes never + * reaches Ohm's law, and sustained compression/shear in the frozen moments + * pumps the orphaned band by induction. Hyper-resistivity (damping ~ eta_H + * k^4) blankets the filtered band while leaving physical scales (k dx < 0.3) + * essentially untouched; size eta_H so eta_H (pi/dx)^4 exceeds the drive. + * + * Clamped to the explicit grad^4 stability limit for this kernel's dt (the + * sub-step dt inside the sub-cycled advance): the discrete grad^2 max + * eigenvalue is 4 D / dx^2, so eta_H (4 D / dx^2)^2 dt <= 1. + */ + Inline auto eta_hyper() const -> real_t { + constexpr real_t ndim = static_cast(static_cast(D)); + return math::min(resist_hyper, + SQR(SQR(dx)) / (SQR(static_cast(4) * ndim) * dt)); + } + + /** + * @brief curl B at the Ee component locations, prefactored into the OUTPUT + * basis of compute_Ee (1D/2D: in-plane comps E/dx, out-of-plane + * physical; 3D: all E/dx). The resistive-vacuum E is +eta times + * these; the hyper-resistive E is -eta_H dx_inv^2 times their + * discrete Laplacian. + */ + Inline void res_curl(const tuple_t& i, + real_t& c0, + real_t& c1, + real_t& c2) const { + if constexpr (D == Dim::_1D) { + const auto i1 = i[0]; + c0 = ZERO; // (curl B)_x = 0 in 1D + c1 = -dx_inv * (Bfs(i1, comp_Bfs + 2) - Bfs(i1 - 1, comp_Bfs + 2)); + c2 = dx_inv * (Bfs(i1, comp_Bfs + 1) - Bfs(i1 - 1, comp_Bfs + 1)); + } else if constexpr (D == Dim::_2D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + c0 = SQR(dx_inv) * + (Bfs(i1, i2, comp_Bfs + 2) - Bfs(i1, i2 - 1, comp_Bfs + 2)); + c1 = -SQR(dx_inv) * + (Bfs(i1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2, comp_Bfs + 2)); + c2 = (Bfs(i1, i2, comp_Bfs + 1) - Bfs(i1 - 1, i2, comp_Bfs + 1)) - + (Bfs(i1, i2, comp_Bfs + 0) - Bfs(i1, i2 - 1, comp_Bfs + 0)); + } else { + const auto i1 = i[0]; + const auto i2 = i[1]; + const auto i3 = i[2]; + c0 = dx_inv * ((Bfs(i1, i2, i3, comp_Bfs + 2) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 2)) - + (Bfs(i1, i2, i3, comp_Bfs + 1) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 1))); + c1 = dx_inv * ((Bfs(i1, i2, i3, comp_Bfs + 0) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 0)) - + (Bfs(i1, i2, i3, comp_Bfs + 2) - + Bfs(i1 - 1, i2, i3, comp_Bfs + 2))); + c2 = dx_inv * ((Bfs(i1, i2, i3, comp_Bfs + 1) - + Bfs(i1 - 1, i2, i3, comp_Bfs + 1)) - + (Bfs(i1, i2, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 0))); + } + } + + /** + * @brief Per-cell limiter on the Hall term. The whistler speed implied at + * the grid cutoff, v_w = d0^2 pi |B| / (dx N), scales with the local + * |B|/N and can exceed the resolvable dx/dt at shock overshoots or + * magnetic cavities (large |B|, small N), where an explicit advance + * is unstable. The limiter scales the Hall term so the implied + * whistler Courant never exceeds `hall_lim`; it is the identity in + * resolvable cells. hall_lim <= 0 disables it. + */ + Inline auto hall_limiter(const tuple_t& i) const -> real_t { + if (hall_lim <= ZERO) { + return ONE; + } + // Worst case (max |B|, min N) over the +/-1 neighborhood the Hall stencil + // reads: a cell-local estimate misses a sharp front where the output + // cell's own B is small but the curl pulls in a much larger neighbor. + real_t bsq = ZERO; + real_t nn; + if constexpr (D == Dim::_1D) { + const auto i1 = i[0]; + nn = NN(i1, comp_NN); + for (int di = -1; di <= 1; ++di) { + const auto ii = i1 + static_cast(di); + // U-basis: b1 = Bx/dx, b2/b3 physical + const real_t b = SQR(dx * Bfs(ii, comp_Bfs + 0)) + + SQR(Bfs(ii, comp_Bfs + 1)) + + SQR(Bfs(ii, comp_Bfs + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, comp_NN)); + } + } else if constexpr (D == Dim::_2D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + nn = NN(i1, i2, comp_NN); + for (int di = -1; di <= 1; ++di) { + for (int dj = -1; dj <= 1; ++dj) { + const auto ii = i1 + static_cast(di); + const auto jj = i2 + static_cast(dj); + // U-basis: b1/b2 = B/dx, b3 physical + const real_t b = SQR(dx * Bfs(ii, jj, comp_Bfs + 0)) + + SQR(dx * Bfs(ii, jj, comp_Bfs + 1)) + + SQR(Bfs(ii, jj, comp_Bfs + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, jj, comp_NN)); + } + } + } else { + const auto i1 = i[0]; + const auto i2 = i[1]; + const auto i3 = i[2]; + nn = NN(i1, i2, i3, comp_NN); + for (int di = -1; di <= 1; ++di) { + for (int dj = -1; dj <= 1; ++dj) { + for (int dk = -1; dk <= 1; ++dk) { + const auto ii = i1 + static_cast(di); + const auto jj = i2 + static_cast(dj); + const auto kk = i3 + static_cast(dk); + // U-basis: all comps physical/dx + const real_t b = SQR(dx * Bfs(ii, jj, kk, comp_Bfs + 0)) + + SQR(dx * Bfs(ii, jj, kk, comp_Bfs + 1)) + + SQR(dx * Bfs(ii, jj, kk, comp_Bfs + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, jj, kk, comp_NN)); + } + } + } + } + // v_w(B, N) = d0^2 pi |B| / (dx N); reduces to the hybrid-CFL formula + // (d0^2/rho0) pi / dx at B = B0 = 1/larmor0, N = 1 + const real_t v_w = SQR(d0) * static_cast(constant::PI) * + math::sqrt(bsq) / (dx * math::max(nn, dens_min)); + // branch instead of min(1, x/y): avoids the 0/0 when B = 0 + return (v_w * dt > hall_lim * dx) ? (hall_lim * dx / (dt * v_w)) : ONE; + } + + /** + * @brief Vacuum cutoff: scales E by min(1, N_raw / dens_min). Below the + * density threshold a cell has no ions to define an E field, so E + * ramps to zero and B is left frozen there, instead of running the + * Ohm's law with a 1/dens_min-amplified right-hand side. The ramp is + * continuous, so cells do not flicker at the threshold and plasma-side + * interface cells keep evolving from their own E. + * + * A hard E = 0 vacuum is only safe when the interface is quiescent. + * When fast plasma sweeps ALONG a vacuum boundary, masking E to zero + * makes an artificial tangential-E discontinuity (a surface current) + * and Faraday pumps B at the interface at rate ~ u/dx. The optional + * resistive vacuum (resist_vac) crossfades E to eta * curl B below the + * threshold instead, so the vacuum diffuses toward a curl-free + * (potential) field and tangential E has a physical continuation. + */ + Inline auto vac_factor(real_t n_raw) const -> real_t { + // safe for dens_min <= 0: the branch is never taken + return (n_raw < dens_min) ? (n_raw / dens_min) : ONE; + } + + Inline void compute_Ee(const tuple_t& i, + real_t& E0, + real_t& E1, + real_t& E2) const { + // Hall/whistler coefficient d_i^2 * Omega_i = d0^2/rho0. Leading minus is + // INTENTIONAL: the (curl B) x B block below is summed into the E bracket, + // which is then multiplied by -1/N. A positive coeff would give + // E_Hall = -(d0^2/rho0)(curl B)xB/N -- the WRONG sign vs the generalized + // Ohm's law (Pegasus eq. 10 / standard Hall-MHD: +(d0^2/rho0)(curl B)xB/N). + // The minus restores the correct sign. (motional & e-pressure unaffected.) + // hall_limiter caps the implied local whistler Courant (see its doc). + const real_t coeff { -SQR(d0) / rho0 * hall_limiter(i) }; + if constexpr (D == Dim::_1D) { + const auto i1 = i[0]; + // Ee* = EMF(N^(n), P^(n), Bf*) + const real_t N0r { NN(i1, comp_NN) }; + const real_t N12r { INV_2 * (NN(i1, comp_NN) + NN(i1 - 1, comp_NN)) }; + const real_t N0 { math::max(N0r, dens_min) }; + const real_t N1 { math::max(N12r, dens_min) }; + const real_t N2 { math::max(N12r, dens_min) }; + + // 1D basis: b1 = Bx/dx, b2/b3 physical; output e1 = Ex/dx, e2/e3 physical + E0 = dx_inv * (-Bfs(i1, comp_Bfs + 1) * PP(i1, comp_PP + 2) + + Bfs(i1, comp_Bfs + 2) * PP(i1, comp_PP + 1)); + E1 = -INV_4 * (Bfs(i1, comp_Bfs + 2) + Bfs(i1 - 1, comp_Bfs + 2)) * + (PP(i1, comp_PP + 0) + PP(i1 - 1, comp_PP + 0)) + + dx * (INV_2) * (PP(i1, comp_PP + 2) + PP(i1 - 1, comp_PP + 2)) * + Bfs(i1, comp_Bfs + 0); + E2 = (INV_4) * (Bfs(i1, comp_Bfs + 1) + Bfs(i1 - 1, comp_Bfs + 1)) * + (PP(i1, comp_PP + 0) + PP(i1 - 1, comp_PP + 0)) - + dx * INV_2 * (PP(i1, comp_PP + 1) + PP(i1 - 1, comp_PP + 1)) * + Bfs(i1, comp_Bfs + 0); + + E0 += coeff * SQR(dx_inv) * + (-INV_2 * (Bfs(i1 + 1, comp_Bfs + 1) - Bfs(i1 - 1, comp_Bfs + 1)) * + Bfs(i1, comp_Bfs + 1) - + INV_2 * (Bfs(i1 + 1, comp_Bfs + 2) - Bfs(i1 - 1, comp_Bfs + 2)) * + Bfs(i1, comp_Bfs + 2)); + E1 += coeff * ((Bfs(i1, comp_Bfs + 1) - Bfs(i1 - 1, comp_Bfs + 1)) * + Bfs(i1, comp_Bfs + 0)); + E2 += coeff * ((Bfs(i1, comp_Bfs + 2) - Bfs(i1 - 1, comp_Bfs + 2)) * + Bfs(i1, comp_Bfs + 0)); + + { + const real_t vac0 { vac_factor(N0r) }; + const real_t vac12 { vac_factor(N12r) }; + E0 *= -vac0 / N0; + E1 *= -vac12 / N1; + E2 *= -vac12 / N2; + if (resist_vac > ZERO or resist_hyper > ZERO) { + real_t c0, c1, c2; + res_curl(i, c0, c1, c2); + if (resist_vac > ZERO) { + // resistive vacuum: crossfade to E = eta * curl B below the + // threshold (see the note at vac_factor) + const real_t eta = eta_vac(); + E1 += (ONE - vac12) * eta * c1; + E2 += (ONE - vac12) * eta * c2; + } + if (resist_hyper > ZERO) { + // hyper-resistivity, everywhere: E -= eta_H grad^2(curl B) + // (see the note at eta_hyper) + real_t l0, l1, l2, r0, r1, r2; + res_curl({ i1 - 1 }, l0, l1, l2); + res_curl({ i1 + 1 }, r0, r1, r2); + const real_t etah = eta_hyper() * SQR(dx_inv); + E1 -= etah * (l1 + r1 - TWO * c1); + E2 -= etah * (l2 + r2 - TWO * c2); + } + } + } + } else if constexpr (D == Dim::_2D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + + const real_t N0r { INV_2 * + (NN(i1, i2, comp_NN) + NN(i1, i2 - 1, comp_NN)) }; + const real_t N1r { INV_2 * + (NN(i1, i2, comp_NN) + NN(i1 - 1, i2, comp_NN)) }; + const real_t N2r { INV_4 * + (NN(i1, i2, comp_NN) + NN(i1, i2 - 1, comp_NN) + + NN(i1 - 1, i2, comp_NN) + + NN(i1 - 1, i2 - 1, comp_NN)) }; + const real_t N0 { math::max(N0r, dens_min) }; + const real_t N1 { math::max(N1r, dens_min) }; + const real_t N2 { math::max(N2r, dens_min) }; + + // 2D basis: b1/b2 = B/dx, b3 physical; output e1/e2 = E/dx, e3 physical + E0 = dx_inv * INV_4 * + (Bfs(i1, i2, comp_Bfs + 2) + Bfs(i1, i2 - 1, comp_Bfs + 2)) * + (PP(i1, i2, comp_PP + 1) + PP(i1, i2 - 1, comp_PP + 1)) - + INV_2 * (PP(i1, i2, comp_PP + 2) + PP(i1, i2 - 1, comp_PP + 2)) * + Bfs(i1, i2, comp_Bfs + 1); + E1 = -dx_inv * INV_4 * + (Bfs(i1, i2, comp_Bfs + 2) + Bfs(i1 - 1, i2, comp_Bfs + 2)) * + (PP(i1, i2, comp_PP + 0) + PP(i1 - 1, i2, comp_PP + 0)) + + (INV_2) * (PP(i1, i2, comp_PP + 2) + PP(i1 - 1, i2, comp_PP + 2)) * + Bfs(i1, i2, comp_Bfs + 0); + E2 = dx * + (-INV_8 * + (Bfs(i1, i2, comp_Bfs + 0) + Bfs(i1, i2 - 1, comp_Bfs + 0)) * + (PP(i1, i2, comp_PP + 1) + PP(i1, i2 - 1, comp_PP + 1) + + PP(i1 - 1, i2, comp_PP + 1) + PP(i1 - 1, i2 - 1, comp_PP + 1)) + + (INV_8) * + (Bfs(i1, i2, comp_Bfs + 1) + Bfs(i1 - 1, i2, comp_Bfs + 1)) * + (PP(i1, i2, comp_PP + 0) + PP(i1, i2 - 1, comp_PP + 0) + + PP(i1 - 1, i2, comp_PP + 0) + PP(i1 - 1, i2 - 1, comp_PP + 0))); + + E0 += + coeff * + (-SQR(dx_inv) * INV_8 * + (Bfs(i1, i2, comp_Bfs + 2) + Bfs(i1, i2 - 1, comp_Bfs + 2)) * + (Bfs(i1 + 1, i2, comp_Bfs + 2) + Bfs(i1 + 1, i2 - 1, comp_Bfs + 2) - + Bfs(i1 - 1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2 - 1, comp_Bfs + 2)) + + (INV_2) * + (Bfs(i1 + 1, i2, comp_Bfs + 0) - Bfs(i1 + 1, i2 - 1, comp_Bfs + 0) + + Bfs(i1, i2, comp_Bfs + 0) - Bfs(i1, i2 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, comp_Bfs + 1) + Bfs(i1 - 1, i2, comp_Bfs + 1)) * + Bfs(i1, i2, comp_Bfs + 1)); + E1 += coeff * + (-SQR(dx_inv) * INV_8 * + (Bfs(i1, i2, comp_Bfs + 2) + Bfs(i1 - 1, i2, comp_Bfs + 2)) * + (Bfs(i1, i2 + 1, comp_Bfs + 2) - Bfs(i1, i2 - 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2 + 1, comp_Bfs + 2) - + Bfs(i1 - 1, i2 - 1, comp_Bfs + 2)) - + INV_2 * + (Bfs(i1, i2 + 1, comp_Bfs + 0) - Bfs(i1, i2 - 1, comp_Bfs + 0) - + Bfs(i1, i2 + 1, comp_Bfs + 1) - Bfs(i1, i2, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2, comp_Bfs + 1)) * + Bfs(i1, i2, comp_Bfs + 0)); + E2 += + coeff * + ((INV_4) * (Bfs(i1, i2, comp_Bfs + 0) + Bfs(i1, i2 - 1, comp_Bfs + 0)) * + (Bfs(i1, i2, comp_Bfs + 2) + Bfs(i1, i2 - 1, comp_Bfs + 2) - + Bfs(i1 - 1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2 - 1, comp_Bfs + 2)) + + (INV_4) * (Bfs(i1, i2, comp_Bfs + 1) + Bfs(i1 - 1, i2, comp_Bfs + 1)) * + (Bfs(i1, i2, comp_Bfs + 2) - Bfs(i1, i2 - 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2 - 1, comp_Bfs + 2))); + + { + const real_t vac0 { vac_factor(N0r) }; + const real_t vac1 { vac_factor(N1r) }; + const real_t vac2 { vac_factor(N2r) }; + E0 *= -vac0 / N0; + E1 *= -vac1 / N1; + E2 *= -vac2 / N2; + if (resist_vac > ZERO or resist_hyper > ZERO) { + real_t c0, c1, c2; + res_curl(i, c0, c1, c2); + if (resist_vac > ZERO) { + // resistive vacuum: crossfade to E = eta * curl B below the + // threshold (see the note at vac_factor) + const real_t eta = eta_vac(); + E0 += (ONE - vac0) * eta * c0; + E1 += (ONE - vac1) * eta * c1; + E2 += (ONE - vac2) * eta * c2; + } + if (resist_hyper > ZERO) { + // hyper-resistivity, everywhere: E -= eta_H grad^2(curl B) + // (see the note at eta_hyper) + real_t w0, w1, w2, e0, e1, e2, s0, s1, s2, n0, n1, n2; + res_curl({ i1 - 1, i2 }, w0, w1, w2); + res_curl({ i1 + 1, i2 }, e0, e1, e2); + res_curl({ i1, i2 - 1 }, s0, s1, s2); + res_curl({ i1, i2 + 1 }, n0, n1, n2); + const real_t etah = eta_hyper() * SQR(dx_inv); + E0 -= etah * (w0 + e0 + s0 + n0 - static_cast(4) * c0); + E1 -= etah * (w1 + e1 + s1 + n1 - static_cast(4) * c1); + E2 -= etah * (w2 + e2 + s2 + n2 - static_cast(4) * c2); + } + } + } + + } else if constexpr (D == Dim::_3D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + const auto i3 = i[2]; + const real_t N0r { INV_4 * (NN(i1, i2, i3, comp_NN) + + NN(i1, i2, i3 - 1, comp_NN) + + NN(i1, i2 - 1, i3, comp_NN) + + NN(i1, i2 - 1, i3 - 1, comp_NN)) }; + const real_t N1r { INV_4 * (NN(i1, i2, i3, comp_NN) + + NN(i1, i2, i3 - 1, comp_NN) + + NN(i1 - 1, i2, i3, comp_NN) + + NN(i1 - 1, i2, i3 - 1, comp_NN)) }; + const real_t N2r { INV_4 * (NN(i1, i2, i3, comp_NN) + + NN(i1, i2 - 1, i3, comp_NN) + + NN(i1 - 1, i2, i3, comp_NN) + + NN(i1 - 1, i2 - 1, i3, comp_NN)) }; + const real_t N0 { math::max(N0r, dens_min) }; + const real_t N1 { math::max(N1r, dens_min) }; + const real_t N2 { math::max(N2r, dens_min) }; + E0 = -INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 1) + Bfs(i1, i2, i3 - 1, comp_Bfs + 1)) * + (PP(i1, i2, i3, comp_PP + 2) + PP(i1, i2, i3 - 1, comp_PP + 2) + + PP(i1, i2 - 1, i3, comp_PP + 2) + + PP(i1, i2 - 1, i3 - 1, comp_PP + 2)) + + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 2) + Bfs(i1, i2 - 1, i3, comp_Bfs + 2)) * + (PP(i1, i2, i3, comp_PP + 1) + PP(i1, i2, i3 - 1, comp_PP + 1) + + PP(i1, i2 - 1, i3, comp_PP + 1) + + PP(i1, i2 - 1, i3 - 1, comp_PP + 1)); + E1 = INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 0) + Bfs(i1, i2, i3 - 1, comp_Bfs + 0)) * + (PP(i1, i2, i3, comp_PP + 2) + PP(i1, i2, i3 - 1, comp_PP + 2) + + PP(i1 - 1, i2, i3, comp_PP + 2) + + PP(i1 - 1, i2, i3 - 1, comp_PP + 2)) - + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 2) + Bfs(i1 - 1, i2, i3, comp_Bfs + 2)) * + (PP(i1, i2, i3, comp_PP + 0) + PP(i1, i2, i3 - 1, comp_PP + 0) + + PP(i1 - 1, i2, i3, comp_PP + 0) + + PP(i1 - 1, i2, i3 - 1, comp_PP + 0)); + E2 = -INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 0) + Bfs(i1, i2 - 1, i3, comp_Bfs + 0)) * + (PP(i1, i2, i3, comp_PP + 1) + PP(i1, i2 - 1, i3, comp_PP + 1) + + PP(i1 - 1, i2, i3, comp_PP + 1) + + PP(i1 - 1, i2 - 1, i3, comp_PP + 1)) + + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 1) + Bfs(i1 - 1, i2, i3, comp_Bfs + 1)) * + (PP(i1, i2, i3, comp_PP + 0) + PP(i1, i2 - 1, i3, comp_PP + 0) + + PP(i1 - 1, i2, i3, comp_PP + 0) + + PP(i1 - 1, i2 - 1, i3, comp_PP + 0)); + + E0 += + coeff * + (INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 1) + Bfs(i1, i2, i3 - 1, comp_Bfs + 1)) * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1 + 1, i2, i3 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 - 1, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0) + Bfs(i1, i2, i3 - 1, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 1) - + Bfs(i1 + 1, i2, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3 - 1, comp_Bfs + 1)) + + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 2) + Bfs(i1, i2 - 1, i3, comp_Bfs + 2)) * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3 - 1, comp_Bfs + 0) + + Bfs(i1 + 1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 - 1, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0) - Bfs(i1, i2, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 2) - + Bfs(i1 + 1, i2 - 1, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3, comp_Bfs + 2))); + E1 += + coeff * + (-INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 0) + Bfs(i1, i2, i3 - 1, comp_Bfs + 0)) * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 0) + + Bfs(i1, i2 + 1, i3 - 1, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3 - 1, comp_Bfs + 0) - + Bfs(i1, i2 + 1, i3, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 - 1, comp_Bfs + 1) - + Bfs(i1, i2, i3, comp_Bfs + 1) - Bfs(i1, i2, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3 - 1, comp_Bfs + 1)) + + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 2) + Bfs(i1 - 1, i2, i3, comp_Bfs + 2)) * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 - 1, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1) - Bfs(i1, i2, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, i3, comp_Bfs + 1) - + Bfs(i1 - 1, i2 + 1, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 1) - + Bfs(i1 - 1, i2, i3 - 1, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3, comp_Bfs + 2) + Bfs(i1, i2 - 1, i3, comp_Bfs + 2) - + Bfs(i1 - 1, i2 + 1, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3, comp_Bfs + 2))); + E2 += + coeff * + (-INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 0) + Bfs(i1, i2 - 1, i3, comp_Bfs + 0)) * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 0) - Bfs(i1, i2, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2 - 1, i3 + 1, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3 - 1, comp_Bfs + 0) - + Bfs(i1, i2, i3 + 1, comp_Bfs + 2) - Bfs(i1, i2, i3, comp_Bfs + 2) - + Bfs(i1, i2 - 1, i3 + 1, comp_Bfs + 2) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3 + 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3, comp_Bfs + 2)) - + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 1) + Bfs(i1 - 1, i2, i3, comp_Bfs + 1)) * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 1) - Bfs(i1, i2, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3 + 1, comp_Bfs + 1) - + Bfs(i1 - 1, i2, i3 - 1, comp_Bfs + 1) - + Bfs(i1, i2, i3 + 1, comp_Bfs + 2) - Bfs(i1, i2, i3, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3, comp_Bfs + 2) - + Bfs(i1 - 1, i2, i3 + 1, comp_Bfs + 2) - + Bfs(i1 - 1, i2, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3 + 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3, comp_Bfs + 2))); + + { + const real_t vac0 { vac_factor(N0r) }; + const real_t vac1 { vac_factor(N1r) }; + const real_t vac2 { vac_factor(N2r) }; + E0 *= -vac0 / N0; + E1 *= -vac1 / N1; + E2 *= -vac2 / N2; + if (resist_vac > ZERO or resist_hyper > ZERO) { + real_t c0, c1, c2; + res_curl(i, c0, c1, c2); + if (resist_vac > ZERO) { + // resistive vacuum: crossfade to E = eta * curl B below the + // threshold (see the note at vac_factor) + const real_t eta = eta_vac(); + E0 += (ONE - vac0) * eta * c0; + E1 += (ONE - vac1) * eta * c1; + E2 += (ONE - vac2) * eta * c2; + } + if (resist_hyper > ZERO) { + // hyper-resistivity, everywhere: E -= eta_H grad^2(curl B) + // (see the note at eta_hyper) + real_t a0, a1, a2, lap0, lap1, lap2; + lap0 = -static_cast(6) * c0; + lap1 = -static_cast(6) * c1; + lap2 = -static_cast(6) * c2; + res_curl({ i1 - 1, i2, i3 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + res_curl({ i1 + 1, i2, i3 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + res_curl({ i1, i2 - 1, i3 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + res_curl({ i1, i2 + 1, i3 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + res_curl({ i1, i2, i3 - 1 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + res_curl({ i1, i2, i3 + 1 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + const real_t etah = eta_hyper() * SQR(dx_inv); + E0 -= etah * lap0; + E1 -= etah * lap1; + E2 -= etah * lap2; + } + } + } + } + } + + Inline void compute_Ec(const tuple_t& i, + real_t& E0, + real_t& E1, + real_t& E2) const { + const real_t coeff_1 { rho0 * gamma_ad * theta }; + // Hall coefficient, same sign convention as `coeff` in compute_Ee above + // (leading minus corrects the (curl B) x B sign; see note there). + // hall_limiter caps the implied local whistler Courant (see its doc). + const real_t coeff_2 { -SQR(d0) / rho0 * hall_limiter(i) }; + if constexpr (D == Dim::_1D) { + const auto i1 = i[0]; + + // floored cell-centered density: the pusher reads Ec raw, so an + // unfloored empty cell would divide by zero into the particle push + const real_t Ncr { NN(i1, comp_NN) }; + const real_t Nc { math::max(Ncr, dens_min) }; + + // Ec* = EMF(N^(n), P^(n), Bc*), where Bc* = interpolate Bf* + // output is PHYSICAL (pusher reads bckp raw); b1 = Bx/dx, b2/b3 physical + E0 = -Bfs(i1, comp_Bfs + 1) * PP(i1, comp_PP + 2) + + Bfs(i1, comp_Bfs + 2) * PP(i1, comp_PP + 1); + E1 = dx * + (INV_2 * Bfs(i1 + 1, comp_Bfs + 0) + + INV_2 * Bfs(i1, comp_Bfs + 0)) * + PP(i1, comp_PP + 2) - + Bfs(i1, comp_Bfs + 2) * PP(i1, comp_PP + 0); + E2 = -dx * + (INV_2 * Bfs(i1 + 1, comp_Bfs + 0) + + INV_2 * Bfs(i1, comp_Bfs + 0)) * + PP(i1, comp_PP + 1) + + Bfs(i1, comp_Bfs + 1) * PP(i1, comp_PP + 0); + + E0 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * INV_2 * + (NN(i1 + 1, comp_NN) - NN(i1 - 1, comp_NN)); + + E0 += coeff_2 * dx_inv * + (-INV_2 * (Bfs(i1 + 1, comp_Bfs + 1) - Bfs(i1 - 1, comp_Bfs + 1)) * + Bfs(i1, comp_Bfs + 1) - + INV_2 * (Bfs(i1 + 1, comp_Bfs + 2) - Bfs(i1 - 1, comp_Bfs + 2)) * + Bfs(i1, comp_Bfs + 2)); + E1 += coeff_2 * INV_4 * + (Bfs(i1 + 1, comp_Bfs + 0) + Bfs(i1, comp_Bfs + 0)) * + (Bfs(i1 + 1, comp_Bfs + 1) - Bfs(i1 - 1, comp_Bfs + 1)); + E2 += coeff_2 * INV_4 * + (Bfs(i1 + 1, comp_Bfs + 0) + Bfs(i1, comp_Bfs + 0)) * + (Bfs(i1 + 1, comp_Bfs + 2) - Bfs(i1 - 1, comp_Bfs + 2)); + + E0 *= -vac_factor(Ncr) / Nc; + E1 *= -vac_factor(Ncr) / Nc; + E2 *= -vac_factor(Ncr) / Nc; + + } else if constexpr (D == Dim::_2D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + + // floored cell-centered density (see 1D note above) + const real_t Ncr { NN(i1, i2, comp_NN) }; + const real_t Nc { math::max(Ncr, dens_min) }; + + // Ec* = EMF(N^(n), P^(n), Bc*), where Bc* = interpolate Bf* + // output is PHYSICAL (pusher reads bckp raw); b1/b2 = B/dx, b3 physical + E0 = -dx * INV_2 * + (Bfs(i1, i2 + 1, comp_Bfs + 1) + Bfs(i1, i2, comp_Bfs + 1)) * + PP(i1, i2, comp_PP + 2) + + Bfs(i1, i2, comp_Bfs + 2) * PP(i1, i2, comp_PP + 1); + E1 = dx * INV_2 * + (Bfs(i1 + 1, i2, comp_Bfs + 0) + Bfs(i1, i2, comp_Bfs + 0)) * + PP(i1, i2, comp_PP + 2) - + Bfs(i1, i2, comp_Bfs + 2) * PP(i1, i2, comp_PP + 0); + E2 = dx * + (-INV_2 * + (Bfs(i1 + 1, i2, comp_Bfs + 0) + Bfs(i1, i2, comp_Bfs + 0)) * + PP(i1, i2, comp_PP + 1) + + INV_2 * + (Bfs(i1, i2 + 1, comp_Bfs + 1) + Bfs(i1, i2, comp_Bfs + 1)) * + PP(i1, i2, comp_PP + 0)); + + E0 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * INV_2 * + (NN(i1 + 1, i2, comp_NN) - NN(i1 - 1, i2, comp_NN)); + E1 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * INV_2 * + (NN(i1, i2 + 1, comp_NN) - NN(i1, i2 - 1, comp_NN)); + E0 += + coeff_2 * + (dx * (INV_8) * + (Bfs(i1, i2 + 1, comp_Bfs + 1) + Bfs(i1, i2, comp_Bfs + 1)) * + (Bfs(i1 + 1, i2 + 1, comp_Bfs + 0) - Bfs(i1 + 1, i2 - 1, comp_Bfs + 0) + + Bfs(i1, i2 + 1, comp_Bfs + 0) - Bfs(i1, i2 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2 + 1, comp_Bfs + 1) - Bfs(i1 + 1, i2, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, comp_Bfs + 1) + Bfs(i1 - 1, i2, comp_Bfs + 1)) - + dx_inv * INV_2 * + (Bfs(i1 + 1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2, comp_Bfs + 2)) * + Bfs(i1, i2, comp_Bfs + 2)); + E1 += + coeff_2 * + (-dx * INV_8 * + (Bfs(i1 + 1, i2, comp_Bfs + 0) + Bfs(i1, i2, comp_Bfs + 0)) * + (Bfs(i1 + 1, i2 + 1, comp_Bfs + 0) - Bfs(i1 + 1, i2 - 1, comp_Bfs + 0) + + Bfs(i1, i2 + 1, comp_Bfs + 0) - Bfs(i1, i2 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2 + 1, comp_Bfs + 1) - Bfs(i1 + 1, i2, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, comp_Bfs + 1) + Bfs(i1 - 1, i2, comp_Bfs + 1)) - + dx_inv * INV_2 * + (Bfs(i1, i2 + 1, comp_Bfs + 2) - Bfs(i1, i2 - 1, comp_Bfs + 2)) * + Bfs(i1, i2, comp_Bfs + 2)); + E2 += coeff_2 * + ((INV_4) * + (Bfs(i1 + 1, i2, comp_Bfs + 0) + Bfs(i1, i2, comp_Bfs + 0)) * + (Bfs(i1 + 1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2, comp_Bfs + 2)) + + (INV_4) * + (Bfs(i1, i2 + 1, comp_Bfs + 1) + Bfs(i1, i2, comp_Bfs + 1)) * + (Bfs(i1, i2 + 1, comp_Bfs + 2) - Bfs(i1, i2 - 1, comp_Bfs + 2))); + + E0 *= -vac_factor(Ncr) / Nc; + E1 *= -vac_factor(Ncr) / Nc; + E2 *= -vac_factor(Ncr) / Nc; + } else if constexpr (D == Dim::_3D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + const auto i3 = i[2]; + + // floored cell-centered density (see 1D note above) + const real_t Ncr { NN(i1, i2, i3, comp_NN) }; + const real_t Nc { math::max(Ncr, dens_min) }; + // Ee* = EMF(N^(n), P^(n), Bf*) + + // Ec* = EMF(N^(n), P^(n), Bc*), where Bc* = interpolate Bf* + // output is PHYSICAL (pusher reads bckp raw); all b comps = B/dx in 3D + E0 = dx * (-INV_2 * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1)) * + PP(i1, i2, i3, comp_PP + 2) + + INV_2 * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2, i3, comp_Bfs + 2)) * + PP(i1, i2, i3, comp_PP + 1)); + E1 = dx * (INV_2 * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0)) * + PP(i1, i2, i3, comp_PP + 2) - + INV_2 * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2, i3, comp_Bfs + 2)) * + PP(i1, i2, i3, comp_PP + 0)); + E2 = dx * (-INV_2 * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0)) * + PP(i1, i2, i3, comp_PP + 1) + + INV_2 * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1)) * + PP(i1, i2, i3, comp_PP + 0)); + + E0 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * + INV_2 * (NN(i1 + 1, i2, i3, comp_NN) - NN(i1 - 1, i2, i3, comp_NN)); + E1 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * + INV_2 * (NN(i1, i2 + 1, i3, comp_NN) - NN(i1, i2 - 1, i3, comp_NN)); + E2 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * + INV_2 * (NN(i1, i2, i3 + 1, comp_NN) - NN(i1, i2, i3 - 1, comp_NN)); + + E0 += coeff_2 * dx * (INV_8 * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1)) * + (Bfs(i1 + 1, i2 + 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 - 1, i3, comp_Bfs + 0) + + Bfs(i1, i2 + 1, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 + 1, i3, comp_Bfs + 1) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 1)) + + INV_8 * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2, i3, comp_Bfs + 2)) * + (Bfs(i1 + 1, i2, i3 + 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2, i3 + 1, comp_Bfs + 0) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3 + 1, comp_Bfs + 2) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 2))); + E1 += coeff_2 * dx * (-INV_8 * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0)) * + (Bfs(i1 + 1, i2 + 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 - 1, i3, comp_Bfs + 0) + + Bfs(i1, i2 + 1, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 + 1, i3, comp_Bfs + 1) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 1)) + + INV_8 * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2, i3, comp_Bfs + 2)) * + (Bfs(i1, i2 + 1, i3 + 1, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 - 1, comp_Bfs + 1) + + Bfs(i1, i2, i3 + 1, comp_Bfs + 1) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 + 1, comp_Bfs + 2) - + Bfs(i1, i2 + 1, i3, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3, comp_Bfs + 2))); + E2 += coeff_2 * dx * (-INV_8 * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0)) * + (Bfs(i1 + 1, i2, i3 + 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2, i3 + 1, comp_Bfs + 0) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3 + 1, comp_Bfs + 2) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 2)) - + INV_8 * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1)) * + (Bfs(i1, i2 + 1, i3 + 1, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 - 1, comp_Bfs + 1) + + Bfs(i1, i2, i3 + 1, comp_Bfs + 1) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 + 1, comp_Bfs + 2) - + Bfs(i1, i2 + 1, i3, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3, comp_Bfs + 2))); + + E0 *= -vac_factor(Ncr) / Nc; + E1 *= -vac_factor(Ncr) / Nc; + E2 *= -vac_factor(Ncr) / Nc; + } + } + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + if constexpr (INIT) { + /// terms of Ee and Ec + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1 }, Eestar0, Eestar1, Eestar2); + + Ee_out(i1, comp_Ee_out + 0) = Eestar0; + Ee_out(i1, comp_Ee_out + 1) = Eestar1; + Ee_out(i1, comp_Ee_out + 2) = Eestar2; + + if constexpr (not EEONLY) { + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1 }, Ecstar0, Ecstar1, Ecstar2); + + Ec_out(i1, comp_Ec_out + 0) = Ecstar0; + Ec_out(i1, comp_Ec_out + 1) = Ecstar1; + Ec_out(i1, comp_Ec_out + 2) = Ecstar2; + } + } else { + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1 }, Eestar0, Eestar1, Eestar2); + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1 }, Ecstar0, Ecstar1, Ecstar2); + + // Ee' = 0.5 * (Ee* + Ee^(n)) + Ee_out(i1, comp_Ee_out + 0) = INV_2 * + (Eestar0 + Ee_in(i1, comp_Ee_in + 0)); + Ee_out(i1, comp_Ee_out + 1) = INV_2 * + (Eestar1 + Ee_in(i1, comp_Ee_in + 1)); + Ee_out(i1, comp_Ee_out + 2) = INV_2 * + (Eestar2 + Ee_in(i1, comp_Ee_in + 2)); + + // Ec' = 0.5 * (Ec* + Ec^(n)) + Ec_out(i1, comp_Ec_out + 0) = INV_2 * (Ecstar0 + Ec(i1, comp_Ec + 0)); + Ec_out(i1, comp_Ec_out + 1) = INV_2 * (Ecstar1 + Ec(i1, comp_Ec + 1)); + Ec_out(i1, comp_Ec_out + 2) = INV_2 * (Ecstar2 + Ec(i1, comp_Ec + 2)); + + // Bc' = 0.5 * (Bc* + Bc^(n)), where Bc* = interpolate Bf* + // (PHYSICAL output: b1 is stored as Bx/dx in 1D, b2/b3 physical) + Bc_out(i1, comp_Bc_out + 0) = dx * INV_2 * + ((INV_2 * Bf(i1 + 1, comp_Bf + 0) + + INV_2 * Bf(i1, comp_Bf + 0)) + + (INV_2 * Bfs(i1 + 1, comp_Bfs + 0) + + INV_2 * Bfs(i1, comp_Bfs + 0))); + + Bc_out(i1, comp_Bc_out + 1) = INV_2 * (Bf(i1, comp_Bf + 1) + + Bfs(i1, comp_Bfs + 1)); + Bc_out(i1, comp_Bc_out + 2) = INV_2 * (Bf(i1, comp_Bf + 2) + + Bfs(i1, comp_Bfs + 2)); + } + } else { + raise::KernelError(HERE, "EMF_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + if constexpr (INIT) { + /// terms of Ee and Ec + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1, i2 }, Eestar0, Eestar1, Eestar2); + + Ee_out(i1, i2, comp_Ee_out + 0) = Eestar0; + Ee_out(i1, i2, comp_Ee_out + 1) = Eestar1; + Ee_out(i1, i2, comp_Ee_out + 2) = Eestar2; + + if constexpr (not EEONLY) { + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1, i2 }, Ecstar0, Ecstar1, Ecstar2); + + Ec_out(i1, i2, comp_Ec_out + 0) = Ecstar0; + Ec_out(i1, i2, comp_Ec_out + 1) = Ecstar1; + Ec_out(i1, i2, comp_Ec_out + 2) = Ecstar2; + } + } else { + // Ee* = EMF(N^(n), P^(n), Bf*) + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1, i2 }, Eestar0, Eestar1, Eestar2); + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1, i2 }, Ecstar0, Ecstar1, Ecstar2); + + // Ee' = 0.5 * (Ee* + Ee^(n)) + Ee_out(i1, i2, comp_Ee_out + 0) = INV_2 * + (Eestar0 + + Ee_in(i1, i2, comp_Ee_in + 0)); + Ee_out(i1, i2, comp_Ee_out + 1) = INV_2 * + (Eestar1 + + Ee_in(i1, i2, comp_Ee_in + 1)); + Ee_out(i1, i2, comp_Ee_out + 2) = INV_2 * + (Eestar2 + + Ee_in(i1, i2, comp_Ee_in + 2)); + + // Ec' = 0.5 * (Ec* + Ec^(n)) + Ec_out(i1, i2, comp_Ec_out + 0) = INV_2 * + (Ecstar0 + Ec(i1, i2, comp_Ec + 0)); + Ec_out(i1, i2, comp_Ec_out + 1) = INV_2 * + (Ecstar1 + Ec(i1, i2, comp_Ec + 1)); + Ec_out(i1, i2, comp_Ec_out + 2) = INV_2 * + (Ecstar2 + Ec(i1, i2, comp_Ec + 2)); + + // Bc' = 0.5 * (Bc* + Bc^(n)), where Bc* = interpolate Bf* + // (PHYSICAL output: b1/b2 are stored as B/dx in 2D, b3 physical) + Bc_out(i1, i2, comp_Bc_out + 0) = dx * INV_4 * + (Bf(i1 + 1, i2, comp_Bf + 0) + + Bf(i1, i2, comp_Bf + 0) + + Bfs(i1 + 1, i2, comp_Bfs + 0) + + Bfs(i1, i2, comp_Bfs + 0)); + + Bc_out(i1, i2, comp_Bc_out + 1) = dx * INV_4 * + (Bf(i1, i2 + 1, comp_Bf + 1) + + Bf(i1, i2, comp_Bf + 1) + + Bfs(i1, i2 + 1, comp_Bfs + 1) + + Bfs(i1, i2, comp_Bfs + 1)); + Bc_out(i1, i2, comp_Bc_out + 2) = INV_2 * (Bf(i1, i2, comp_Bf + 2) + + Bfs(i1, i2, comp_Bfs + 2)); + } + } else { + raise::KernelError(HERE, "EMF_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + if constexpr (INIT) { + /// terms of Ee and Ec + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1, i2, i3 }, Eestar0, Eestar1, Eestar2); + + Ee_out(i1, i2, i3, comp_Ee_out + 0) = Eestar0; + Ee_out(i1, i2, i3, comp_Ee_out + 1) = Eestar1; + Ee_out(i1, i2, i3, comp_Ee_out + 2) = Eestar2; + + if constexpr (not EEONLY) { + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1, i2, i3 }, Ecstar0, Ecstar1, Ecstar2); + + Ec_out(i1, i2, i3, comp_Ec_out + 0) = Ecstar0; + Ec_out(i1, i2, i3, comp_Ec_out + 1) = Ecstar1; + Ec_out(i1, i2, i3, comp_Ec_out + 2) = Ecstar2; + } + } else { + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1, i2, i3 }, Eestar0, Eestar1, Eestar2); + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1, i2, i3 }, Ecstar0, Ecstar1, Ecstar2); + + // Ee' = 0.5 * (Ee* + Ee^(n)) + Ee_out(i1, i2, i3, comp_Ee_out + 0) = INV_2 * + (Eestar0 + + Ee_in(i1, i2, i3, comp_Ee_in + 0)); + Ee_out(i1, i2, i3, comp_Ee_out + 1) = INV_2 * + (Eestar1 + + Ee_in(i1, i2, i3, comp_Ee_in + 1)); + Ee_out(i1, i2, i3, comp_Ee_out + 2) = INV_2 * + (Eestar2 + + Ee_in(i1, i2, i3, comp_Ee_in + 2)); + + // Ec' = 0.5 * (Ec* + Ec^(n)) + Ec_out(i1, i2, i3, comp_Ec_out + 0) = INV_2 * + (Ecstar0 + + Ec(i1, i2, i3, comp_Ec + 0)); + Ec_out(i1, i2, i3, comp_Ec_out + 1) = INV_2 * + (Ecstar1 + + Ec(i1, i2, i3, comp_Ec + 1)); + Ec_out(i1, i2, i3, comp_Ec_out + 2) = INV_2 * + (Ecstar2 + + Ec(i1, i2, i3, comp_Ec + 2)); + + // Bc' = 0.5 * (Bc* + Bc^(n)), where Bc* = interpolate Bf* + // (PHYSICAL output: all b comps are stored as B/dx in 3D) + Bc_out(i1, i2, i3, comp_Bc_out + 0) = dx * INV_4 * + (Bf(i1 + 1, i2, i3, comp_Bf + 0) + + Bf(i1, i2, i3, comp_Bf + 0) + + Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0)); + + Bc_out(i1, i2, i3, comp_Bc_out + 1) = dx * INV_4 * + (Bf(i1, i2 + 1, i3, comp_Bf + 1) + + Bf(i1, i2, i3, comp_Bf + 1) + + Bfs(i1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1)); + Bc_out(i1, i2, i3, comp_Bc_out + 2) = dx * INV_4 * + (Bf(i1, i2, i3 + 1, comp_Bf + 2) + + Bf(i1, i2, i3, comp_Bf + 2) + + Bfs(i1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2, i3, comp_Bfs + 2)); + } + } else { + raise::KernelError(HERE, "EMF_kernel: 3D implementation called for D != 3"); + } + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_EMF_HPP diff --git a/src/kernels/hybrid/faraday.hpp b/src/kernels/hybrid/faraday.hpp new file mode 100644 index 000000000..efd26023c --- /dev/null +++ b/src/kernels/hybrid/faraday.hpp @@ -0,0 +1,123 @@ +#ifndef KERNELS_HYBRID_FARADAY_HPP +#define KERNELS_HYBRID_FARADAY_HPP + +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/numeric.h" + +namespace kernel::hybrid { + + /** + * @note Fields are stored in the contravariant (Idx::U) code basis of the + * Minkowski metric: components with index i <= D carry a 1/dx + * (h_ii = dx^2), out-of-plane components are physical (h_ii = 1). The + * index-space curl therefore needs per-component coefficients: + * 1D: dt/dx for b2, b3 (from physical e2, e3); b1 untouched + * 2D: dt/dx^2 for b1, b2 (from physical e3); dt for b3 (from e1, e2) + * 3D: dt/dx for all components + */ + // NIN: number of components of Bin (6 = em, 3 = cur). Bin == Bout in-place is + // safe: the update reads only the cell's own B (the stencil is on Ein). + template + class Faraday_kernel { + ndfield_t Ein; + ndfield_t Bin; + ndfield_t Bout; + + const uint8_t comp_Ein; + const uint8_t comp_Bin; + const uint8_t comp_Bout; + + const real_t dt; + const real_t dx_inv; + + public: + Faraday_kernel(const ndfield_t& Ein, + const ndfield_t& Bin, + ndfield_t& Bout, + uint8_t comp_Ein, + uint8_t comp_Bin, + uint8_t comp_Bout, + real_t dt, + real_t dx) + : Ein { Ein } + , Bin { Bin } + , Bout { Bout } + , comp_Ein { comp_Ein } + , comp_Bin { comp_Bin } + , comp_Bout { comp_Bout } + , dt { dt } + , dx_inv { ONE / dx } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + // B_x is constant in 1D (dB_x/dt = -(curl E)_x = 0). Bout here is a + // SEPARATE scratch buffer (cur = Bf*/Bf** for the predictor pushes), + // NOT in-place on `em`, and is zero-initialized -- so B_x must be + // copied through, else the subsequent EMF (Ohm's-law Hall/motional + // terms) would read Bf*_x = 0 instead of B_x^n. + const real_t coeff { dt * dx_inv }; + Bout(i1, comp_Bout + 0) = Bin(i1, comp_Bin + 0); + Bout(i1, comp_Bout + 1) = Bin(i1, comp_Bin + 1) - + coeff * (-Ein(i1 + 1, comp_Ein + 2) + + Ein(i1, comp_Ein + 2)); + Bout(i1, comp_Bout + 2) = Bin(i1, comp_Bin + 2) - + coeff * (Ein(i1 + 1, comp_Ein + 1) - + Ein(i1, comp_Ein + 1)); + } else { + raise::KernelError(HERE, "Faraday_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + const real_t coeff1 { dt * SQR(dx_inv) }; // in-plane b from physical e3 + const real_t coeff2 { dt }; // out-of-plane b3 from e1/dx, e2/dx + Bout(i1, i2, comp_Bout + 0) = Bin(i1, i2, comp_Bin + 0) - + coeff1 * (Ein(i1, i2 + 1, comp_Ein + 2) - + Ein(i1, i2, comp_Ein + 2)); + Bout(i1, i2, comp_Bout + 1) = Bin(i1, i2, comp_Bin + 1) - + coeff1 * (-Ein(i1 + 1, i2, comp_Ein + 2) + + Ein(i1, i2, comp_Ein + 2)); + Bout(i1, i2, comp_Bout + 2) = Bin(i1, i2, comp_Bin + 2) - + coeff2 * (-Ein(i1, i2 + 1, comp_Ein + 0) + + Ein(i1, i2, comp_Ein + 0) + + Ein(i1 + 1, i2, comp_Ein + 1) - + Ein(i1, i2, comp_Ein + 1)); + } else { + raise::KernelError(HERE, "Faraday_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + const real_t coeff { dt * dx_inv }; // all comps stored as physical/dx + Bout(i1, i2, i3, comp_Bout + 0) = Bin(i1, i2, i3, comp_Bin + 0) - + coeff * + (-Ein(i1, i2, i3 + 1, comp_Ein + 1) + + Ein(i1, i2, i3, comp_Ein + 1) + + Ein(i1, i2 + 1, i3, comp_Ein + 2) - + Ein(i1, i2, i3, comp_Ein + 2)); + Bout(i1, i2, i3, comp_Bout + 1) = Bin(i1, i2, i3, comp_Bin + 1) - + coeff * + (Ein(i1, i2, i3 + 1, comp_Ein + 0) - + Ein(i1, i2, i3, comp_Ein + 0) - + Ein(i1 + 1, i2, i3, comp_Ein + 2) + + Ein(i1, i2, i3, comp_Ein + 2)); + Bout(i1, i2, i3, comp_Bout + 2) = Bin(i1, i2, i3, comp_Bin + 2) - + coeff * + (-Ein(i1, i2 + 1, i3, comp_Ein + 0) + + Ein(i1, i2, i3, comp_Ein + 0) + + Ein(i1 + 1, i2, i3, comp_Ein + 1) - + Ein(i1, i2, i3, comp_Ein + 1)); + } else { + raise::KernelError(HERE, "Faraday_kernel: 3D implementation called for D != 3"); + } + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_FARADAY_HPP diff --git a/src/kernels/hybrid/moments_filter.hpp b/src/kernels/hybrid/moments_filter.hpp new file mode 100644 index 000000000..06e0c14e2 --- /dev/null +++ b/src/kernels/hybrid/moments_filter.hpp @@ -0,0 +1,118 @@ +#ifndef KERNELS_HYBRID_MOMENTS_FILTER_HPP +#define KERNELS_HYBRID_MOMENTS_FILTER_HPP + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/numeric.h" + +namespace kernel::hybrid { + using namespace ntt; + + /** + * Binomial (1-2-1) smoothing of the deposited hybrid moments. + * + * The Ohm's-law E is built directly from the deposited number density N and + * momentum density V (aux::3 and aux::012). With cold, fast beams the per-cell + * shot noise in V (~ v_drift / sqrt(ppc)) is injected straight into E via the + * motional (-u x B) and Hall (J x B / N) terms and drives a grid-scale + * numerical instability. Smoothing the moments before the field solve removes + * the grid-scale (Nyquist) noise, exactly as current filtering does in the + * explicit PIC engines. + * + * Filters comps 0..3 (V0, V1, V2, N) of the 6-component `array` (aux), reading + * from `buffer` (a copy of `array`). Cartesian, interior stencil only: the + * i +/- 1 reads reach into the ghost layer, which the caller refills (periodic + * / MPI) between passes via CommunicateFields(AUX). N_GHOSTS >= 1 is required + * (always true for SHAPE_ORDER >= 0). + * + * @note Conductor/reflecting walls are NOT special-cased here (unlike + * DigitalFilter_kernel); the plain stencil smooths against whatever the + * wall ghost moments hold. Hybrid wall setups should revisit this if they + * enable filtering. + */ + template + class MomentsFilter_kernel { + ndfield_t array; + const ndfield_t buffer; + + static constexpr unsigned short NCOMP = 4; // V0, V1, V2, N + + public: + MomentsFilter_kernel(const ndfield_t& array, + const ndfield_t& buffer) + : array { array } + , buffer { buffer } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { +#pragma unroll + for (auto c { 0u }; c < NCOMP; ++c) { + array(i1, c) = INV_2 * buffer(i1, c) + + INV_4 * (buffer(i1 - 1, c) + buffer(i1 + 1, c)); + } + } else { + raise::KernelError(HERE, "MomentsFilter_kernel: 1D called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { +#pragma unroll + for (auto c { 0u }; c < NCOMP; ++c) { + array(i1, i2, c) = INV_4 * buffer(i1, i2, c) + + INV_8 * (buffer(i1 - 1, i2, c) + + buffer(i1 + 1, i2, c) + + buffer(i1, i2 - 1, c) + + buffer(i1, i2 + 1, c)) + + INV_16 * (buffer(i1 - 1, i2 - 1, c) + + buffer(i1 + 1, i2 + 1, c) + + buffer(i1 - 1, i2 + 1, c) + + buffer(i1 + 1, i2 - 1, c)); + } + } else { + raise::KernelError(HERE, "MomentsFilter_kernel: 2D called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { +#pragma unroll + for (auto c { 0u }; c < NCOMP; ++c) { + array(i1, i2, i3, c) = + INV_8 * buffer(i1, i2, i3, c) + + INV_16 * (buffer(i1 - 1, i2, i3, c) + buffer(i1 + 1, i2, i3, c) + + buffer(i1, i2 - 1, i3, c) + buffer(i1, i2 + 1, i3, c) + + buffer(i1, i2, i3 - 1, c) + buffer(i1, i2, i3 + 1, c)) + + INV_32 * (buffer(i1 - 1, i2 - 1, i3, c) + + buffer(i1 + 1, i2 + 1, i3, c) + + buffer(i1 - 1, i2 + 1, i3, c) + + buffer(i1 + 1, i2 - 1, i3, c) + + buffer(i1, i2 - 1, i3 - 1, c) + + buffer(i1, i2 + 1, i3 + 1, c) + + buffer(i1, i2 - 1, i3 + 1, c) + + buffer(i1, i2 + 1, i3 - 1, c) + + buffer(i1 - 1, i2, i3 - 1, c) + + buffer(i1 + 1, i2, i3 + 1, c) + + buffer(i1 - 1, i2, i3 + 1, c) + + buffer(i1 + 1, i2, i3 - 1, c)) + + INV_64 * (buffer(i1 - 1, i2 - 1, i3 - 1, c) + + buffer(i1 + 1, i2 + 1, i3 + 1, c) + + buffer(i1 - 1, i2 + 1, i3 + 1, c) + + buffer(i1 + 1, i2 - 1, i3 - 1, c) + + buffer(i1 - 1, i2 - 1, i3 + 1, c) + + buffer(i1 + 1, i2 + 1, i3 - 1, c) + + buffer(i1 - 1, i2 + 1, i3 - 1, c) + + buffer(i1 + 1, i2 - 1, i3 + 1, c)); + } + } else { + raise::KernelError(HERE, "MomentsFilter_kernel: 3D called for D != 3"); + } + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_MOMENTS_FILTER_HPP diff --git a/src/kernels/hybrid/pusher.hpp b/src/kernels/hybrid/pusher.hpp new file mode 100644 index 000000000..def32452e --- /dev/null +++ b/src/kernels/hybrid/pusher.hpp @@ -0,0 +1,623 @@ +/** + * @file kernels/hybrid/pusher.hpp + * @brief Non-relativistic modified-Boris ion pusher (+ fused moment deposit) + * for the HYBRID engine + * @implements + * - kernel::hybrid::PushMode + * - kernel::hybrid::PusherContext + * - kernel::hybrid::Pusher_kernel<> + * @namespaces: + * - kernel::hybrid:: + * + * Implements the Pegasus modified-Boris integrator (Kunz, Stone & Bai 2014, + * arXiv:1311.4865, eq. 12) for the base case sigma = varpi = 0. One push advances + * a particle from t^(n) to t^(n+1) using fields centered at t^(n+1/2): + * + * (12a) x* = x^(n) + (dt/2) v^(n) // first half drift + * (12b) v^- = v^(n) + (dt/2) c Ec // first electric half-kick + * (12c) v^+ = v^- + rotation(Bc) // Boris rotation (gamma = 1) + * (12d) v^(n+1) = v^+ + (dt/2) c Ec // second electric half-kick + * (12e) x^(n+1) = x* + (dt/2) v^(n+1) // second half drift + * + * with the coupling c = (q/m), folded with omegaB0*dt/2 into `normalized_dt_half`. + * Ec, Bc are *cell-centered*, *time-centered at n+1/2* fields (the `bckp` buffer: + * Ec in comps 0..2, Bc in comps 3..5), interpolated at the predicted position x* + * with the SAME shape function used by the deposit (paper ยง3.2). + * + * FUSED DEPOSIT. Immediately after advancing a particle, the kernel deposits the + * ion number density N -> aux comp 3 and momentum density V = sum(w*m*v) -> + * aux comps 0..2, with NO Lorentz factor (non-relativistic). This avoids a second + * pass through the particle table and lets the transient predictor produce its + * predicted moments WITHOUT writing the particle arrays (so no save/restore of + * x^(n), v^(n) is needed โ€” both pushes start from the stored state). + * + * The template `PushMode Mode` selects the three uses (Pegasus Fig. 2): + * - MomentsOnly : deposit N,V from the stored x^(n), v^(n); no push, no store. + * Used once at step 0 to seed aux with N^(0), V^(0). + * - Predictor : push in registers (Fig. 2 steps 7+8), deposit predicted + * N', V'; do NOT write the particle arrays, no particle BCs. + * - Corrector : push (Fig. 2 step 12), deposit final N^(n+1), V^(n+1), AND + * write back x^(n+1), v^(n+1) (+ i/dx_prev) and apply particle BCs. + * + * @note NON-RELATIVISTIC. ux1/ux2/ux3 store the 3-velocity v; gamma == 1 everywhere + * (no 1/sqrt(1+v^2) in the rotation, no dt/gamma in the drift). + * @note Cartesian Minkowski only. 1D gather/deposit are implemented; 2D/3D are + * stubs (the hybrid EMF solver is itself 1D-only for now). + * @see PIC/hybrid/pusher.md for the full plan and the correctness traps. + */ + +#ifndef KERNELS_HYBRID_PUSHER_HPP +#define KERNELS_HYBRID_PUSHER_HPP + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/numeric.h" + +#include "framework/containers/particles.h" +#include "kernels/particle_shapes.hpp" +#include "kernels/pushers/context.h" // kernel::sr::PusherBoundaries (BC flags) + +#if defined(MPI_ENABLED) + #include "arch/mpi_tags.h" +#endif + +namespace kernel::hybrid { + using namespace ntt; + + enum class PushMode { + MomentsOnly, // deposit only (step-0 seed); no push, no store + Predictor, // push in registers + deposit; no store, no BC + Corrector // push + deposit + store-back + BC + }; + + /** + * @brief Lean parameter bundle for the hybrid ion pusher / deposit. + * @note mass/charge live on the species, not on `ParticleArrays`, so they are + * passed in here (cf. kernel::sr::PusherContext). + */ + struct PusherContext { + const float mass, charge; + const real_t dt; + const real_t omegaB0; // = 1 / larmor0 + const real_t inv_n0; // = 1 / scales.n0 (number-density normalization) + const bool use_weights; + const int ni1, ni2, ni3; // # active cells per direction (periodic wrap) + + PusherContext(float mass, + float charge, + real_t dt, + real_t omegaB0, + real_t inv_n0, + bool use_weights, + int ni1, + int ni2, + int ni3) + : mass { mass } + , charge { charge } + , dt { dt } + , omegaB0 { omegaB0 } + , inv_n0 { inv_n0 } + , use_weights { use_weights } + , ni1 { ni1 } + , ni2 { ni2 } + , ni3 { ni3 } {} + }; + + /** + * @tparam M Metric (Cartesian Minkowski for the hybrid engine). + * @tparam Mode MomentsOnly / Predictor / Corrector โ€” see file header. + */ + template + struct Pusher_kernel { + static constexpr auto D = M::Dim; + static_assert(M::CoordType == Coord::Cartesian, + "hybrid pusher: Cartesian (Minkowski) metric only"); + + // shape order and half-window; gather and deposit MUST share the cell-centering + static constexpr unsigned short O = SHAPE_ORDER; + static constexpr int window = (SHAPE_ORDER + 1) / 2; + + const PusherContext ctx; + const kernel::sr::PusherBoundaries bc; + ParticleArrays particles; + + // read-only, cell-centered, time-centered (n+1/2) fields: + // Ec -> comps 0..2, Bc -> comps 3..5 (the `bckp` buffer); unused in MomentsOnly + const randacc_ndfield_t EB; + // scatter view over `aux`: V -> comps 0..2, N -> comp 3 + scatter_ndfield_t moments; + + const M metric; + + // coefficients (precomputed once) + const real_t normalized_dt_half; // 1/2 * (q/m) * omegaB0 * dt (E-kick & rotation) + const real_t dt_half; // 1/2 * dt (each half drift) + + Pusher_kernel(const PusherContext& pusher_ctx, + const kernel::sr::PusherBoundaries& pusher_boundaries, + ParticleArrays& pusher_arrays, + const randacc_ndfield_t& EB, + const scatter_ndfield_t& moments, + const M& metric) + : ctx { pusher_ctx } + , bc { pusher_boundaries } + , particles { pusher_arrays } + , EB { EB } + , moments { moments } + , metric { metric } + , normalized_dt_half { HALF * (ctx.charge / ctx.mass) * ctx.omegaB0 * ctx.dt } + , dt_half { HALF * ctx.dt } {} + + // ........................................................................ + // main per-particle update + // ........................................................................ + Inline void operator()(prtlidx_t p) const { + if (particles.tag(p) != ParticleTag::alive) { + return; + } + + // load start-of-step state x^(n), v^(n); ux holds the 3-velocity v + const int i1n { particles.i1(p) }; + const prtldx_t dx1n { particles.dx1(p) }; + int i2n { 0 }, i3n { 0 }; + prtldx_t dx2n { ZERO }, dx3n { ZERO }; + if constexpr (D == Dim::_2D || D == Dim::_3D) { + i2n = particles.i2(p); + dx2n = particles.dx2(p); + } + if constexpr (D == Dim::_3D) { + i3n = particles.i3(p); + dx3n = particles.dx3(p); + } + + // working state in registers (real_t offset avoids float drift across substeps) + int i1 { i1n }, i2 { i2n }, i3 { i3n }; + real_t dx1 { static_cast(dx1n) }; + real_t dx2 { static_cast(dx2n) }; + real_t dx3 { static_cast(dx3n) }; + vec_t v { particles.ux1(p), particles.ux2(p), particles.ux3(p) }; + + if constexpr (Mode != PushMode::MomentsOnly) { + // (12a) first half drift to the predicted position x* + halfDrift(i1, dx1, i2, dx2, i3, dx3, v); + // interpolate cell-centered Ec, Bc at x* (Cartesian: already in XYZ basis) + vec_t e0 { ZERO }, b0 { ZERO }; + gather(i1, dx1, i2, dx2, i3, dx3, e0, b0); + // (12b-d) electric half-kick / Boris rotation / electric half-kick + velocityPush(e0, b0, v); + // (12e) second half drift to x^(n+1) + halfDrift(i1, dx1, i2, dx2, i3, dx3, v); + } + + // fused moment deposit at the advanced (or, for MomentsOnly, the stored) state + deposit(p, i1, dx1, i2, dx2, i3, dx3, v); + + // corrector commits the accepted move -> store back + particle boundaries + if constexpr (Mode == PushMode::Corrector) { + if constexpr (D == Dim::_1D || D == Dim::_2D || D == Dim::_3D) { + particles.i1_prev(p) = i1n; + particles.dx1_prev(p) = dx1n; + particles.i1(p) = i1; + particles.dx1(p) = static_cast(dx1); + } + if constexpr (D == Dim::_2D || D == Dim::_3D) { + particles.i2_prev(p) = i2n; + particles.dx2_prev(p) = dx2n; + particles.i2(p) = i2; + particles.dx2(p) = static_cast(dx2); + } + if constexpr (D == Dim::_3D) { + particles.i3_prev(p) = i3n; + particles.dx3_prev(p) = dx3n; + particles.i3(p) = i3; + particles.dx3(p) = static_cast(dx3); + } + particles.ux1(p) = v[0]; + particles.ux2(p) = v[1]; + particles.ux3(p) = v[2]; + boundaryConditions(p); + } + } + + // ........................................................................ + // velocity update โ€” eq. (12b)-(12d), gamma == 1 + // e0,b0 enter as the raw interpolated fields and are scaled here. + // ........................................................................ + Inline void velocityPush(vec_t& e0, + vec_t& b0, + vec_t& v) const { + e0[0] *= normalized_dt_half; + e0[1] *= normalized_dt_half; + e0[2] *= normalized_dt_half; + b0[0] *= normalized_dt_half; // t = (dt/2) c B (NO relativistic 1/gamma!) + b0[1] *= normalized_dt_half; + b0[2] *= normalized_dt_half; + + // (12b) v^- = v + (dt/2) c E + v[0] += e0[0]; + v[1] += e0[1]; + v[2] += e0[2]; + + // (12c) Boris rotation: v^+ = v^- + s (v^- + v^- x t) x t, s = 2/(1+|t|^2) + const real_t s { TWO / (ONE + NORM_SQR(b0[0], b0[1], b0[2])) }; + const vec_t vp { + (v[0] + CROSS_x1(v[0], v[1], v[2], b0[0], b0[1], b0[2])) * s, + (v[1] + CROSS_x2(v[0], v[1], v[2], b0[0], b0[1], b0[2])) * s, + (v[2] + CROSS_x3(v[0], v[1], v[2], b0[0], b0[1], b0[2])) * s + }; + v[0] += CROSS_x1(vp[0], vp[1], vp[2], b0[0], b0[1], b0[2]); + v[1] += CROSS_x2(vp[0], vp[1], vp[2], b0[0], b0[1], b0[2]); + v[2] += CROSS_x3(vp[0], vp[1], vp[2], b0[0], b0[1], b0[2]); + + // (12d) v^(n+1) = v^+ + (dt/2) c E + v[0] += e0[0]; + v[1] += e0[1]; + v[2] += e0[2]; + } + + // ........................................................................ + // half-step drift: x += (dt/2) v, expressed as i/dx increments. + // transform converts the Cartesian velocity to a coordinate + // (cell-fraction) velocity by dividing by the cell size dx. + // ........................................................................ + Inline void halfDrift(int& i1, + real_t& dx1, + int& i2, + real_t& dx2, + int& i3, + real_t& dx3, + const vec_t& v) const { + coord_t xp { ZERO }; + xCoord(i1, dx1, i2, dx2, i3, dx3, xp); + if constexpr (D == Dim::_1D || D == Dim::_2D || D == Dim::_3D) { + dx1 += metric.template transform<1, Idx::XYZ, Idx::U>(xp, v[0]) * dt_half; + rebucket(i1, dx1); + } + if constexpr (D == Dim::_2D || D == Dim::_3D) { + dx2 += metric.template transform<2, Idx::XYZ, Idx::U>(xp, v[1]) * dt_half; + rebucket(i2, dx2); + } + if constexpr (D == Dim::_3D) { + dx3 += metric.template transform<3, Idx::XYZ, Idx::U>(xp, v[2]) * dt_half; + rebucket(i3, dx3); + } + } + + // single-cell rebucketing of (i, dx); valid for |displacement| < 1 cell (CFL) + static Inline void rebucket(int& i, real_t& dx) { + const int shift { static_cast(dx >= ONE) - static_cast(dx < ZERO) }; + i += shift; + dx -= static_cast(shift); + } + + // fill the code-coordinate position xp from the working i/dx state + // (only consumed by metric.transform, position-independent for Minkowski) + Inline void xCoord(int i1, + real_t dx1, + int i2, + real_t dx2, + int i3, + real_t dx3, + coord_t& xp) const { + if constexpr (D == Dim::_1D || D == Dim::_2D || D == Dim::_3D) { + xp[0] = static_cast(i1) + dx1; + } + if constexpr (D == Dim::_2D || D == Dim::_3D) { + xp[1] = static_cast(i2) + dx2; + } + if constexpr (D == Dim::_3D) { + xp[2] = static_cast(i3) + dx3; + } + } + + // valid cell-index guards. The field arrays span ni + 2*N_GHOSTS in each + // direction; a particle that overshoots a boundary by more than one ghost + // layer in a single step (e.g. bulk-flow Courant > 1, or near a reflecting + // wall before the BC is applied) would otherwise gather/scatter past the + // array end โ€” an out-of-bounds atomic that page-faults on GPU. Skipping the + // out-of-range cell keeps the kernel safe; the lost edge contribution is + // negligible and the particle is wrapped/reflected by the boundary handler. + Inline auto inX1(int c) const -> bool { + return (c >= 0) && (c < ctx.ni1 + 2 * static_cast(N_GHOSTS)); + } + Inline auto inX2(int c) const -> bool { + return (c >= 0) && (c < ctx.ni2 + 2 * static_cast(N_GHOSTS)); + } + Inline auto inX3(int c) const -> bool { + return (c >= 0) && (c < ctx.ni3 + 2 * static_cast(N_GHOSTS)); + } + + // ........................................................................ + // cell-centered field gather at (i, dx) โ€” transpose of the moment deposit. + // ........................................................................ + Inline void gather(int i1, + real_t dx1, + int i2, + real_t dx2, + int i3, + real_t dx3, + vec_t& e0, + vec_t& b0) const { + if constexpr (D == Dim::_1D) { + for (int di1 { -window }; di1 <= window; ++di1) { + const int c { i1 + di1 + static_cast(N_GHOSTS) }; + if (not inX1(c)) { + continue; + } + const real_t S { prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + e0[0] += S * EB(c, 0); + e0[1] += S * EB(c, 1); + e0[2] += S * EB(c, 2); + b0[0] += S * EB(c, 3); + b0[1] += S * EB(c, 4); + b0[2] += S * EB(c, 5); + } + } else if constexpr (D == Dim::_2D) { + for (int di2 { -window }; di2 <= window; ++di2) { + const int c2 { i2 + di2 + static_cast(N_GHOSTS) }; + if (not inX2(c2)) { + continue; + } + const real_t sx2 { prtl_shape::particle_shape( + math::abs(dx2 - (static_cast(di2) + HALF))) }; + for (int di1 { -window }; di1 <= window; ++di1) { + const int c1 { i1 + di1 + static_cast(N_GHOSTS) }; + if (not inX1(c1)) { + continue; + } + const real_t S { sx2 * prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + e0[0] += S * EB(c1, c2, 0); + e0[1] += S * EB(c1, c2, 1); + e0[2] += S * EB(c1, c2, 2); + b0[0] += S * EB(c1, c2, 3); + b0[1] += S * EB(c1, c2, 4); + b0[2] += S * EB(c1, c2, 5); + } + } + } else if constexpr (D == Dim::_3D) { + for (int di3 { -window }; di3 <= window; ++di3) { + const int c3 { i3 + di3 + static_cast(N_GHOSTS) }; + if (not inX3(c3)) { + continue; + } + const real_t sx3 { prtl_shape::particle_shape( + math::abs(dx3 - (static_cast(di3) + HALF))) }; + for (int di2 { -window }; di2 <= window; ++di2) { + const int c2 { i2 + di2 + static_cast(N_GHOSTS) }; + if (not inX2(c2)) { + continue; + } + const real_t sx23 { sx3 * prtl_shape::particle_shape( + math::abs(dx2 - (static_cast(di2) + HALF))) }; + for (int di1 { -window }; di1 <= window; ++di1) { + const int c1 { i1 + di1 + static_cast(N_GHOSTS) }; + if (not inX1(c1)) { + continue; + } + const real_t S { sx23 * prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + e0[0] += S * EB(c1, c2, c3, 0); + e0[1] += S * EB(c1, c2, c3, 1); + e0[2] += S * EB(c1, c2, c3, 2); + b0[0] += S * EB(c1, c2, c3, 3); + b0[1] += S * EB(c1, c2, c3, 4); + b0[2] += S * EB(c1, c2, c3, 5); + } + } + } + } + } + + // ........................................................................ + // fused moment deposit: N -> aux::3, V = m*v -> aux::0..2 (non-relativistic). + // Cell-centered, shape-weighted, transpose of `gather`. + // ........................................................................ + Inline void deposit(prtlidx_t p, + int i1, + real_t dx1, + int i2, + real_t dx2, + int i3, + real_t dx3, + const vec_t& v) const { + real_t w { ctx.inv_n0 }; + if constexpr (D == Dim::_1D) { + w /= metric.sqrt_det_h({ static_cast(i1) + HALF }); + } else if constexpr (D == Dim::_2D) { + w /= metric.sqrt_det_h( + { static_cast(i1) + HALF, static_cast(i2) + HALF }); + } else if constexpr (D == Dim::_3D) { + w /= metric.sqrt_det_h({ static_cast(i1) + HALF, + static_cast(i2) + HALF, + static_cast(i3) + HALF }); + } + if (ctx.use_weights) { + w *= particles.weight(p); + } + const real_t cN { w }; + const real_t cV0 { w * ctx.mass * v[0] }; + const real_t cV1 { w * ctx.mass * v[1] }; + const real_t cV2 { w * ctx.mass * v[2] }; + + auto buff = moments.access(); + if constexpr (D == Dim::_1D) { + for (int di1 { -window }; di1 <= window; ++di1) { + const int c { i1 + di1 + static_cast(N_GHOSTS) }; + if (not inX1(c)) { + continue; + } + const real_t S { prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + buff(c, 0) += cV0 * S; + buff(c, 1) += cV1 * S; + buff(c, 2) += cV2 * S; + buff(c, 3) += cN * S; + } + } else if constexpr (D == Dim::_2D) { + for (int di2 { -window }; di2 <= window; ++di2) { + const int c2 { i2 + di2 + static_cast(N_GHOSTS) }; + if (not inX2(c2)) { + continue; + } + const real_t sx2 { prtl_shape::particle_shape( + math::abs(dx2 - (static_cast(di2) + HALF))) }; + for (int di1 { -window }; di1 <= window; ++di1) { + const int c1 { i1 + di1 + static_cast(N_GHOSTS) }; + if (not inX1(c1)) { + continue; + } + const real_t S { sx2 * prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + buff(c1, c2, 0) += cV0 * S; + buff(c1, c2, 1) += cV1 * S; + buff(c1, c2, 2) += cV2 * S; + buff(c1, c2, 3) += cN * S; + } + } + } else if constexpr (D == Dim::_3D) { + for (int di3 { -window }; di3 <= window; ++di3) { + const int c3 { i3 + di3 + static_cast(N_GHOSTS) }; + if (not inX3(c3)) { + continue; + } + const real_t sx3 { prtl_shape::particle_shape( + math::abs(dx3 - (static_cast(di3) + HALF))) }; + for (int di2 { -window }; di2 <= window; ++di2) { + const int c2 { i2 + di2 + static_cast(N_GHOSTS) }; + if (not inX2(c2)) { + continue; + } + const real_t sx23 { sx3 * prtl_shape::particle_shape( + math::abs(dx2 - (static_cast(di2) + HALF))) }; + for (int di1 { -window }; di1 <= window; ++di1) { + const int c1 { i1 + di1 + static_cast(N_GHOSTS) }; + if (not inX1(c1)) { + continue; + } + const real_t S { sx23 * prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + buff(c1, c2, c3, 0) += cV0 * S; + buff(c1, c2, c3, 1) += cV1 * S; + buff(c1, c2, c3, 2) += cV2 * S; + buff(c1, c2, c3, 3) += cN * S; + } + } + } + } + } + + // ........................................................................ + // particle boundaries โ€” periodic / absorb / reflect (Cartesian Minkowski, + // so a reflection just negates the corresponding 3-velocity component), plus + // the MPI leaving-direction tagging. Modeled on kernels/pushers/sr.hpp:659-814. + // ........................................................................ + Inline void boundaryConditions(prtlidx_t p) const { + if constexpr (D == Dim::_1D || D == Dim::_2D || D == Dim::_3D) { + if (particles.i1(p) < 0) { + if (bc.is_periodic_i1min) { + particles.i1(p) += ctx.ni1; + particles.i1_prev(p) += ctx.ni1; + } else if (bc.is_absorb_i1min) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i1min) { + particles.i1(p) = 0; + particles.dx1(p) = ONE - particles.dx1(p); + particles.ux1(p) = -particles.ux1(p); + } + } else if (particles.i1(p) >= ctx.ni1) { + if (bc.is_periodic_i1max) { + particles.i1(p) -= ctx.ni1; + particles.i1_prev(p) -= ctx.ni1; + } else if (bc.is_absorb_i1max) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i1max) { + particles.i1(p) = ctx.ni1 - 1; + particles.dx1(p) = ONE - particles.dx1(p); + particles.ux1(p) = -particles.ux1(p); + } + } + } + if constexpr (D == Dim::_2D || D == Dim::_3D) { + if (particles.i2(p) < 0) { + if (bc.is_periodic_i2min) { + particles.i2(p) += ctx.ni2; + particles.i2_prev(p) += ctx.ni2; + } else if (bc.is_absorb_i2min) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i2min) { + particles.i2(p) = 0; + particles.dx2(p) = ONE - particles.dx2(p); + particles.ux2(p) = -particles.ux2(p); + } + } else if (particles.i2(p) >= ctx.ni2) { + if (bc.is_periodic_i2max) { + particles.i2(p) -= ctx.ni2; + particles.i2_prev(p) -= ctx.ni2; + } else if (bc.is_absorb_i2max) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i2max) { + particles.i2(p) = ctx.ni2 - 1; + particles.dx2(p) = ONE - particles.dx2(p); + particles.ux2(p) = -particles.ux2(p); + } + } + } + if constexpr (D == Dim::_3D) { + if (particles.i3(p) < 0) { + if (bc.is_periodic_i3min) { + particles.i3(p) += ctx.ni3; + particles.i3_prev(p) += ctx.ni3; + } else if (bc.is_absorb_i3min) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i3min) { + particles.i3(p) = 0; + particles.dx3(p) = ONE - particles.dx3(p); + particles.ux3(p) = -particles.ux3(p); + } + } else if (particles.i3(p) >= ctx.ni3) { + if (bc.is_periodic_i3max) { + particles.i3(p) -= ctx.ni3; + particles.i3_prev(p) -= ctx.ni3; + } else if (bc.is_absorb_i3max) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i3max) { + particles.i3(p) = ctx.ni3 - 1; + particles.dx3(p) = ONE - particles.dx3(p); + particles.ux3(p) = -particles.ux3(p); + } + } + } +#if defined(MPI_ENABLED) + // tag the particle with the direction it leaves the local subdomain so the + // metadomain particle exchange ships it to the right neighbor + if constexpr (D == Dim::_1D) { + particles.tag(p) = mpi::SendTag(particles.tag(p), + particles.i1(p) < 0, + particles.i1(p) >= ctx.ni1); + } else if constexpr (D == Dim::_2D) { + particles.tag(p) = mpi::SendTag(particles.tag(p), + particles.i1(p) < 0, + particles.i1(p) >= ctx.ni1, + particles.i2(p) < 0, + particles.i2(p) >= ctx.ni2); + } else if constexpr (D == Dim::_3D) { + particles.tag(p) = mpi::SendTag(particles.tag(p), + particles.i1(p) < 0, + particles.i1(p) >= ctx.ni1, + particles.i2(p) < 0, + particles.i2(p) >= ctx.ni2, + particles.i3(p) < 0, + particles.i3(p) >= ctx.ni3); + } +#endif + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_PUSHER_HPP diff --git a/src/kernels/hybrid/subcycle.hpp b/src/kernels/hybrid/subcycle.hpp new file mode 100644 index 000000000..97005c503 --- /dev/null +++ b/src/kernels/hybrid/subcycle.hpp @@ -0,0 +1,198 @@ +/** + * @file kernels/hybrid/subcycle.hpp + * @brief Support kernels for the sub-cycled (Pegasus-style) magnetic-field + * advance of the HYBRID engine. + * @implements + * - kernel::hybrid::FieldCombine_kernel<> -> dst = a*dst + b*src over a + * 3-component group (a = 0, b = 1 is a copy). Launched over the FULL + * extent so valid ghosts stay valid. + * - kernel::hybrid::VwMax_kernel<> -> parallel_reduce functor for + * max over active cells of the local whistler speed at the grid cutoff, + * v_w = d0^2 pi |B| / (dx max(N, dens_min)) -- used to pick the number of + * field sub-steps each advance needs. + * @namespaces: + * - kernel::hybrid:: + */ + +#ifndef KERNELS_HYBRID_SUBCYCLE_HPP +#define KERNELS_HYBRID_SUBCYCLE_HPP + +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/numeric.h" + +namespace kernel::hybrid { + using namespace ntt; + + template + struct FieldCombine_kernel { + ndfield_t Dst; + ndfield_t Src; + const real_t a, b; + const uint8_t cd, cs; + + FieldCombine_kernel(ndfield_t& Dst, + const ndfield_t& Src, + real_t a, + real_t b, + uint8_t cd, + uint8_t cs) + : Dst { Dst } + , Src { Src } + , a { a } + , b { b } + , cd { cd } + , cs { cs } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + for (uint8_t c = 0; c < 3; ++c) { + Dst(i1, cd + c) = a * Dst(i1, cd + c) + b * Src(i1, cs + c); + } + } else { + raise::KernelError( + HERE, + "FieldCombine_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + for (uint8_t c = 0; c < 3; ++c) { + Dst(i1, i2, cd + c) = a * Dst(i1, i2, cd + c) + b * Src(i1, i2, cs + c); + } + } else { + raise::KernelError( + HERE, + "FieldCombine_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + for (uint8_t c = 0; c < 3; ++c) { + Dst(i1, i2, i3, cd + c) = a * Dst(i1, i2, i3, cd + c) + + b * Src(i1, i2, i3, cs + c); + } + } else { + raise::KernelError( + HERE, + "FieldCombine_kernel: 3D implementation called for D != 3"); + } + } + }; + + /** + * @brief max-reduce of v_w(B, N) = d0^2 pi |B|_phys / (dx max(N, dens_min)) + * over active cells. B is read from a 6-component field (em) at + * comp_B (U-basis: in-plane comps are physical/dx); N from aux comp 3. + * STENCIL-AWARE like EMF_kernel::hall_limiter: |B| and N are taken + * from the worst cells of the +/-1 neighborhood -- at a void / + * overshoot interface the large B and the small N sit in DIFFERENT + * cells, and a same-cell estimate misses exactly the edge whistler + * speed the discrete Hall operator feels. + */ + template + struct VwMax_kernel { + ndfield_t Bfld; + ndfield_t NN; + const uint8_t comp_B; + const uint8_t comp_NN; + const real_t d0, dens_min, dx; + + VwMax_kernel(const ndfield_t& Bfld, + const ndfield_t& NN, + uint8_t comp_B, + uint8_t comp_NN, + real_t d0, + real_t dens_min, + real_t dx) + : Bfld { Bfld } + , NN { NN } + , comp_B { comp_B } + , comp_NN { comp_NN } + , d0 { d0 } + , dens_min { dens_min } + , dx { dx } {} + + Inline void operator()(cellidx_t i1, real_t& lmax) const { + if constexpr (D == Dim::_1D) { + real_t bsq = ZERO; + real_t nn = NN(i1, comp_NN); + for (int di = -1; di <= 1; ++di) { + const auto ii = i1 + static_cast(di); + // U-basis: b1 = Bx/dx, b2/b3 physical + const real_t b = SQR(dx * Bfld(ii, comp_B + 0)) + + SQR(Bfld(ii, comp_B + 1)) + + SQR(Bfld(ii, comp_B + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, comp_NN)); + } + const real_t vw = SQR(d0) * static_cast(constant::PI) * + math::sqrt(bsq) / (dx * math::max(nn, dens_min)); + lmax = math::max(lmax, vw); + } else { + raise::KernelError(HERE, + "VwMax_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, real_t& lmax) const { + if constexpr (D == Dim::_2D) { + real_t bsq = ZERO; + real_t nn = NN(i1, i2, comp_NN); + for (int di = -1; di <= 1; ++di) { + for (int dj = -1; dj <= 1; ++dj) { + const auto ii = i1 + static_cast(di); + const auto jj = i2 + static_cast(dj); + // U-basis: b1/b2 = B/dx, b3 physical + const real_t b = SQR(dx * Bfld(ii, jj, comp_B + 0)) + + SQR(dx * Bfld(ii, jj, comp_B + 1)) + + SQR(Bfld(ii, jj, comp_B + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, jj, comp_NN)); + } + } + const real_t vw = SQR(d0) * static_cast(constant::PI) * + math::sqrt(bsq) / (dx * math::max(nn, dens_min)); + lmax = math::max(lmax, vw); + } else { + raise::KernelError(HERE, + "VwMax_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3, real_t& lmax) const { + if constexpr (D == Dim::_3D) { + real_t bsq = ZERO; + real_t nn = NN(i1, i2, i3, comp_NN); + for (int di = -1; di <= 1; ++di) { + for (int dj = -1; dj <= 1; ++dj) { + for (int dk = -1; dk <= 1; ++dk) { + const auto ii = i1 + static_cast(di); + const auto jj = i2 + static_cast(dj); + const auto kk = i3 + static_cast(dk); + // U-basis: all comps physical/dx + const real_t b = SQR(dx * Bfld(ii, jj, kk, comp_B + 0)) + + SQR(dx * Bfld(ii, jj, kk, comp_B + 1)) + + SQR(dx * Bfld(ii, jj, kk, comp_B + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, jj, kk, comp_NN)); + } + } + } + const real_t vw = SQR(d0) * static_cast(constant::PI) * + math::sqrt(bsq) / (dx * math::max(nn, dens_min)); + lmax = math::max(lmax, vw); + } else { + raise::KernelError(HERE, + "VwMax_kernel: 3D implementation called for D != 3"); + } + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_SUBCYCLE_HPP diff --git a/src/kernels/hybrid/wall_bcs.hpp b/src/kernels/hybrid/wall_bcs.hpp new file mode 100644 index 000000000..000e7daa2 --- /dev/null +++ b/src/kernels/hybrid/wall_bcs.hpp @@ -0,0 +1,380 @@ +/** + * @file kernels/hybrid/wall_bcs.hpp + * @brief Reflecting-wall (perfect-conductor) boundary kernels for the HYBRID + * engine, x1 walls only. + * @implements + * - kernel::hybrid::WallEdgeE_kernel<> -> edge-E: E_tan = 0 on the wall + * plane + odd/even mirror into the ghosts + * - kernel::hybrid::WallFaceB_kernel<> -> face-B: even (zero-gradient) + * mirror into the ghosts; the wall-plane B_n is NOT touched + * - kernel::hybrid::WallMoments_kernel<> -> cell-centered moments (aux): + * additive fold of the ghost deposit tails (image plasma) + mirror fill + * - kernel::hybrid::WallBckp_kernel<> -> cell-centered Ec/Bc gather + * fields: conductor-sign mirror fill of the ghosts + * @namespaces: + * - kernel::hybrid:: + * + * Conventions (P = true is the +x1 wall): + * - i_edge is the wall-plane NODE index: i_min(x1) for -x1, i_max(x1) for + * +x1. Cell i is bounded by nodes i and i+1, so the ghost cells are + * i_edge-1-k (minus) / i_edge+k (plus), k = 0..N_GHOSTS-1. + * - Edge-E staggering in x1: comp +0 (E_x) is x1-cell-centered, comps +1/+2 + * (E_y/E_z) sit on x1 nodes -- at i1 == i_edge they lie ON the wall plane. + * - Face-B staggering in x1: comp +0 (B_x) sits on x1 nodes, comps +1/+2 are + * x1-cell-centered. + * + * Physics: + * - Perfect conductor: E_tan = 0 on the wall plane. Faraday then freezes the + * wall-plane B_n automatically (dB_n/dt = -(curl E)_n involves only + * in-plane E_tan derivatives), which is the correct condition for an + * OBLIQUE background field (B_n stays at its initial value). The wall-plane + * B_n must therefore never be overwritten -- in particular NOT zeroed, + * which is only valid when the initial B_n is zero (perpendicular shocks) + * and otherwise plants a div(B) monopole layer at the wall. + * - B ghosts are filled with an even (zero-gradient) mirror so the Hall term + * (curl B) x B sees no fake wall current sheet. Without this, scratch + * buffers whose wall ghosts stay zero (e.g. Bf*, Bf** in `cur`) produce a + * spurious J ~ B/dx at the wall every substage. + * - Moments: a reflecting wall is realized by image plasma (u_x -> -u_x). + * The shape-function tails a near-wall particle deposits into the ghost + * cells are exactly the image particle's contribution to the active cells: + * fold them back with V_x sign-flipped (V_y, V_z, N unchanged), then fill + * the ghosts with the same-sign mirror of the active cells for the EMF / + * filter stencils. Without the fold the wall-cell density is undercounted + * and 1/N in Ohm's law is overestimated. + * - bckp (cell-centered Ec/Bc the pusher gathers): conductor-sign mirror, + * E_x even, E_y/E_z odd (-> E_tan ~ 0 at the wall), B even. Without it the + * gather mixes in zero ghosts and near-wall particles feel ~50% fields. + */ + +#ifndef KERNELS_HYBRID_WALL_BCS_HPP +#define KERNELS_HYBRID_WALL_BCS_HPP + +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" + +namespace kernel::hybrid { + using namespace ntt; + + /** + * @brief E_tan = 0 on the wall plane + conductor mirror into the ghosts. + * @tparam D dimension + * @tparam P true for the +x1 wall, false for the -x1 wall + * Launch range: i1 in [0, N_GHOSTS + 1) for -x1, [0, N_GHOSTS) for +x1 + * (i1 == 0 is the wall plane, i1 >= 1 the ghost layers); other indices span + * the full transverse extent. + */ + template + struct WallEdgeE_kernel { + ndfield_t Fld; + const ncells_t i_edge; + const uint8_t c0; + + WallEdgeE_kernel(ndfield_t& Fld, ncells_t i_edge, uint8_t c0) + : Fld { Fld } + , i_edge { i_edge } + , c0 { c0 } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + if (i1 == 0) { + Fld(i_edge, c0 + 1) = ZERO; + Fld(i_edge, c0 + 2) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, c0 + 0) = Fld(i_edge + i1 - 1, c0 + 0); + Fld(i_edge - i1, c0 + 1) = -Fld(i_edge + i1, c0 + 1); + Fld(i_edge - i1, c0 + 2) = -Fld(i_edge + i1, c0 + 2); + } else { + Fld(i_edge + i1 - 1, c0 + 0) = Fld(i_edge - i1, c0 + 0); + Fld(i_edge + i1, c0 + 1) = -Fld(i_edge - i1, c0 + 1); + Fld(i_edge + i1, c0 + 2) = -Fld(i_edge - i1, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallEdgeE_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + if (i1 == 0) { + Fld(i_edge, i2, c0 + 1) = ZERO; + Fld(i_edge, i2, c0 + 2) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, c0 + 0) = Fld(i_edge + i1 - 1, i2, c0 + 0); + Fld(i_edge - i1, i2, c0 + 1) = -Fld(i_edge + i1, i2, c0 + 1); + Fld(i_edge - i1, i2, c0 + 2) = -Fld(i_edge + i1, i2, c0 + 2); + } else { + Fld(i_edge + i1 - 1, i2, c0 + 0) = Fld(i_edge - i1, i2, c0 + 0); + Fld(i_edge + i1, i2, c0 + 1) = -Fld(i_edge - i1, i2, c0 + 1); + Fld(i_edge + i1, i2, c0 + 2) = -Fld(i_edge - i1, i2, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallEdgeE_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + if (i1 == 0) { + Fld(i_edge, i2, i3, c0 + 1) = ZERO; + Fld(i_edge, i2, i3, c0 + 2) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, c0 + 0) = Fld(i_edge + i1 - 1, i2, i3, c0 + 0); + Fld(i_edge - i1, i2, i3, c0 + 1) = -Fld(i_edge + i1, i2, i3, c0 + 1); + Fld(i_edge - i1, i2, i3, c0 + 2) = -Fld(i_edge + i1, i2, i3, c0 + 2); + } else { + Fld(i_edge + i1 - 1, i2, i3, c0 + 0) = Fld(i_edge - i1, i2, i3, c0 + 0); + Fld(i_edge + i1, i2, i3, c0 + 1) = -Fld(i_edge - i1, i2, i3, c0 + 1); + Fld(i_edge + i1, i2, i3, c0 + 2) = -Fld(i_edge - i1, i2, i3, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallEdgeE_kernel: 3D implementation called for D != 3"); + } + } + }; + + /** + * @brief Even (zero-gradient) mirror of face-B into the wall ghosts; the + * wall-plane B_n is left untouched (frozen by Faraday + E_tan = 0). + * @tparam D dimension + * @tparam P true for the +x1 wall + * @tparam N number of components of the field array (6 for em, 3 for cur) + * Launch range: as WallEdgeE_kernel (i1 == 0 is a no-op kept for range + * symmetry). + */ + template + struct WallFaceB_kernel { + ndfield_t Fld; + const ncells_t i_edge; + const uint8_t c0; + + WallFaceB_kernel(ndfield_t& Fld, ncells_t i_edge, uint8_t c0) + : Fld { Fld } + , i_edge { i_edge } + , c0 { c0 } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + if (i1 != 0) { + if constexpr (not P) { + Fld(i_edge - i1, c0 + 0) = Fld(i_edge + i1, c0 + 0); + Fld(i_edge - i1, c0 + 1) = Fld(i_edge + i1 - 1, c0 + 1); + Fld(i_edge - i1, c0 + 2) = Fld(i_edge + i1 - 1, c0 + 2); + } else { + Fld(i_edge + i1, c0 + 0) = Fld(i_edge - i1, c0 + 0); + Fld(i_edge + i1 - 1, c0 + 1) = Fld(i_edge - i1, c0 + 1); + Fld(i_edge + i1 - 1, c0 + 2) = Fld(i_edge - i1, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallFaceB_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + if (i1 != 0) { + if constexpr (not P) { + Fld(i_edge - i1, i2, c0 + 0) = Fld(i_edge + i1, i2, c0 + 0); + Fld(i_edge - i1, i2, c0 + 1) = Fld(i_edge + i1 - 1, i2, c0 + 1); + Fld(i_edge - i1, i2, c0 + 2) = Fld(i_edge + i1 - 1, i2, c0 + 2); + } else { + Fld(i_edge + i1, i2, c0 + 0) = Fld(i_edge - i1, i2, c0 + 0); + Fld(i_edge + i1 - 1, i2, c0 + 1) = Fld(i_edge - i1, i2, c0 + 1); + Fld(i_edge + i1 - 1, i2, c0 + 2) = Fld(i_edge - i1, i2, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallFaceB_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + if (i1 != 0) { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, c0 + 0) = Fld(i_edge + i1, i2, i3, c0 + 0); + Fld(i_edge - i1, i2, i3, c0 + 1) = Fld(i_edge + i1 - 1, i2, i3, c0 + 1); + Fld(i_edge - i1, i2, i3, c0 + 2) = Fld(i_edge + i1 - 1, i2, i3, c0 + 2); + } else { + Fld(i_edge + i1, i2, i3, c0 + 0) = Fld(i_edge - i1, i2, i3, c0 + 0); + Fld(i_edge + i1 - 1, i2, i3, c0 + 1) = Fld(i_edge - i1, i2, i3, c0 + 1); + Fld(i_edge + i1 - 1, i2, i3, c0 + 2) = Fld(i_edge - i1, i2, i3, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallFaceB_kernel: 3D implementation called for D != 3"); + } + } + }; + + /** + * @brief Image-plasma treatment of the deposited moments at an x1 wall. + * FOLD = true: aux(active) += s_c * aux(ghost) -- folds the ghost + * deposit tails back as the image-particle contribution (must run + * exactly once per deposit, after SynchronizeFields(AUX)). + * FOLD = false: aux(ghost) = s_c * aux(active) -- idempotent mirror + * fill for stencils that read the wall ghosts. + * Signs: V_x odd (comp 0), V_y/V_z/N even (comps 1..3). + * @tparam D dimension + * @tparam P true for the +x1 wall + * Launch range: i1 in [0, N_GHOSTS) = ghost layer; other indices: active + * extent for FOLD (corner tails are already remapped by the transverse + * sync), full extent for fill. + */ + template + struct WallMoments_kernel { + ndfield_t Fld; + const ncells_t i_edge; + + WallMoments_kernel(ndfield_t& Fld, ncells_t i_edge) + : Fld { Fld } + , i_edge { i_edge } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + if constexpr (FOLD) { + Fld(ia, 0) += -Fld(ig, 0); + Fld(ia, 1) += Fld(ig, 1); + Fld(ia, 2) += Fld(ig, 2); + Fld(ia, 3) += Fld(ig, 3); + } else { + Fld(ig, 0) = -Fld(ia, 0); + Fld(ig, 1) = Fld(ia, 1); + Fld(ig, 2) = Fld(ia, 2); + Fld(ig, 3) = Fld(ia, 3); + } + } else { + raise::KernelError(HERE, + "WallMoments_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + if constexpr (FOLD) { + Fld(ia, i2, 0) += -Fld(ig, i2, 0); + Fld(ia, i2, 1) += Fld(ig, i2, 1); + Fld(ia, i2, 2) += Fld(ig, i2, 2); + Fld(ia, i2, 3) += Fld(ig, i2, 3); + } else { + Fld(ig, i2, 0) = -Fld(ia, i2, 0); + Fld(ig, i2, 1) = Fld(ia, i2, 1); + Fld(ig, i2, 2) = Fld(ia, i2, 2); + Fld(ig, i2, 3) = Fld(ia, i2, 3); + } + } else { + raise::KernelError(HERE, + "WallMoments_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + if constexpr (FOLD) { + Fld(ia, i2, i3, 0) += -Fld(ig, i2, i3, 0); + Fld(ia, i2, i3, 1) += Fld(ig, i2, i3, 1); + Fld(ia, i2, i3, 2) += Fld(ig, i2, i3, 2); + Fld(ia, i2, i3, 3) += Fld(ig, i2, i3, 3); + } else { + Fld(ig, i2, i3, 0) = -Fld(ia, i2, i3, 0); + Fld(ig, i2, i3, 1) = Fld(ia, i2, i3, 1); + Fld(ig, i2, i3, 2) = Fld(ia, i2, i3, 2); + Fld(ig, i2, i3, 3) = Fld(ia, i2, i3, 3); + } + } else { + raise::KernelError(HERE, + "WallMoments_kernel: 3D implementation called for D != 3"); + } + } + }; + + /** + * @brief Conductor-sign mirror fill of the cell-centered gather fields + * (bckp: Ec in comps 0..2, Bc in comps 3..5) into the x1-wall ghosts: + * E_x even, E_y/E_z odd, B even. + * @tparam D dimension + * @tparam P true for the +x1 wall + * Launch range: i1 in [0, N_GHOSTS); other indices: full extent. + */ + template + struct WallBckp_kernel { + ndfield_t Fld; + const ncells_t i_edge; + + WallBckp_kernel(ndfield_t& Fld, ncells_t i_edge) + : Fld { Fld } + , i_edge { i_edge } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + Fld(ig, 0) = Fld(ia, 0); + Fld(ig, 1) = -Fld(ia, 1); + Fld(ig, 2) = -Fld(ia, 2); + Fld(ig, 3) = Fld(ia, 3); + Fld(ig, 4) = Fld(ia, 4); + Fld(ig, 5) = Fld(ia, 5); + } else { + raise::KernelError(HERE, + "WallBckp_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + Fld(ig, i2, 0) = Fld(ia, i2, 0); + Fld(ig, i2, 1) = -Fld(ia, i2, 1); + Fld(ig, i2, 2) = -Fld(ia, i2, 2); + Fld(ig, i2, 3) = Fld(ia, i2, 3); + Fld(ig, i2, 4) = Fld(ia, i2, 4); + Fld(ig, i2, 5) = Fld(ia, i2, 5); + } else { + raise::KernelError(HERE, + "WallBckp_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + Fld(ig, i2, i3, 0) = Fld(ia, i2, i3, 0); + Fld(ig, i2, i3, 1) = -Fld(ia, i2, i3, 1); + Fld(ig, i2, i3, 2) = -Fld(ia, i2, i3, 2); + Fld(ig, i2, i3, 3) = Fld(ia, i2, i3, 3); + Fld(ig, i2, i3, 4) = Fld(ia, i2, i3, 4); + Fld(ig, i2, i3, 5) = Fld(ia, i2, i3, 5); + } else { + raise::KernelError(HERE, + "WallBckp_kernel: 3D implementation called for D != 3"); + } + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_WALL_BCS_HPP diff --git a/src/kernels/injectors.hpp b/src/kernels/injectors.hpp index 5d7904cd9..7b0be32de 100644 --- a/src/kernels/injectors.hpp +++ b/src/kernels/injectors.hpp @@ -17,6 +17,7 @@ #include "arch/kokkos_aliases.h" #include "traits/archetypes.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/error.h" #include "utils/numeric.h" @@ -222,17 +223,21 @@ namespace kernel { if constexpr (M::CoordType == Coord::Cartesian) { energy_dist_1(x_Ph, v1); energy_dist_2(x_Ph, v2); - } else if constexpr (S == SimEngine::SRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCartesianBasis) { coord_t x_Cd_ { ZERO }; x_Cd_[0] = x_Cd[0]; x_Cd_[1] = x_Cd[1]; - x_Cd_[2] = ZERO; // phi = 0 + if constexpr (::traits::engine::HasImplicitPhiCoordinate) { + x_Cd_[2] = ZERO; // phi = 0 + } else { + x_Cd_[2] = x_Cd[2]; + } vec_t v_Ph { ZERO }; energy_dist_1(x_Ph, v_Ph); metric.template transform_xyz(x_Cd_, v_Ph, v1); energy_dist_2(x_Ph, v_Ph); metric.template transform_xyz(x_Cd_, v_Ph, v2); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { vec_t v_Ph { ZERO }; energy_dist_1(x_Ph, v_Ph); metric.template transform(x_Cd, v_Ph, v1); @@ -288,6 +293,157 @@ namespace kernel { } }; // struct UniformInjector_kernel + /** + * @brief Single-species variant of UniformInjector_kernel + * @note injects one species per cell (e.g. the lone ion species of the hybrid + * engine), where the charge-neutral pair injector does not apply + */ + template ED> + struct UniformInjectorSingle_kernel { + + array_t i1s, i2s, i3s; + array_t dx1s, dx2s, dx3s; + array_t ux1s, ux2s, ux3s; + array_t phis; + array_t weights; + array_t tags; + array_t pldis; + + const npart_t offset; + const npart_t domain_idx, cntr; + const bool use_tracking; + const M metric; + const array_t xi_min, xi_max; + const ED energy_dist; + const real_t inv_V0; + random_number_pool_t random_pool; + + UniformInjectorSingle_kernel(Particles& species, + npart_t domain_idx, + const M& metric, + const array_t& xi_min, + const array_t& xi_max, + const ED& energy_dist, + real_t inv_V0, + random_number_pool_t& random_pool) + : i1s { species.i1 } + , i2s { species.i2 } + , i3s { species.i3 } + , dx1s { species.dx1 } + , dx2s { species.dx2 } + , dx3s { species.dx3 } + , ux1s { species.ux1 } + , ux2s { species.ux2 } + , ux3s { species.ux3 } + , phis { species.phi } + , weights { species.weight } + , tags { species.tag } + , pldis { species.pld_i } + , offset { species.npart() } + , domain_idx { domain_idx } + , cntr { species.counter() } + , use_tracking { species.use_tracking() } + , metric { metric } + , xi_min { xi_min } + , xi_max { xi_max } + , energy_dist { energy_dist } + , inv_V0 { inv_V0 } + , random_pool { random_pool } { + if (use_tracking) { +#if !defined(MPI_ENABLED) + raise::ErrorIf(species.pld_i.extent(1) < 1, + "Particle tracking is enabled but the " + "particle integer payload size is less " + "than 1", + HERE); +#else + raise::ErrorIf(species.pld_i.extent(1) < 2, + "Particle tracking is enabled but the " + "particle integer payload size is less " + "than 2", + HERE); +#endif + } + } + + Inline void operator()(prtlidx_t p) const { + coord_t x_Cd { ZERO }; + tuple_t xi_Cd { 0 }; + tuple_t dxi_Cd { static_cast(0) }; + vec_t v { ZERO }; + { // generate a random coordinate + auto rand_gen = random_pool.get_state(); + if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or + M::Dim == Dim::_3D) { + x_Cd[0] = xi_min(0) + Random(rand_gen) * (xi_max(0) - xi_min(0)); + xi_Cd[0] = static_cast(x_Cd[0]); + dxi_Cd[0] = static_cast(x_Cd[0] - xi_Cd[0]); + } + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + x_Cd[1] = xi_min(1) + Random(rand_gen) * (xi_max(1) - xi_min(1)); + xi_Cd[1] = static_cast(x_Cd[1]); + dxi_Cd[1] = static_cast(x_Cd[1] - xi_Cd[1]); + } + if constexpr (M::Dim == Dim::_3D) { + x_Cd[2] = xi_min(2) + Random(rand_gen) * (xi_max(2) - xi_min(2)); + xi_Cd[2] = static_cast(x_Cd[2]); + dxi_Cd[2] = static_cast(x_Cd[2] - xi_Cd[2]); + } + random_pool.free_state(rand_gen); + } + { // generate the velocity + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + if constexpr (M::CoordType == Coord::Cartesian) { + energy_dist(x_Ph, v); + } else if constexpr (::traits::engine::VelocitiesInCartesianBasis) { + coord_t x_Cd_ { ZERO }; + x_Cd_[0] = x_Cd[0]; + x_Cd_[1] = x_Cd[1]; + if constexpr (::traits::engine::HasImplicitPhiCoordinate) { + x_Cd_[2] = ZERO; // phi = 0 + } else { + x_Cd_[2] = x_Cd[2]; + } + vec_t v_Ph { ZERO }; + energy_dist(x_Ph, v_Ph); + metric.template transform_xyz(x_Cd_, v_Ph, v); + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { + vec_t v_Ph { ZERO }; + energy_dist(x_Ph, v_Ph); + metric.template transform(x_Cd, v_Ph, v); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); + } + } + real_t weight = ONE; + if constexpr (M::CoordType != Coord::Cartesian) { + const auto sqrt_det_h = metric.sqrt_det_h(x_Cd); + weight = sqrt_det_h * inv_V0; + } + // clang-format off + if (not use_tracking) { + InjectParticle( + p + offset, + i1s, i2s, i3s, + dx1s, dx2s, dx3s, + ux1s, ux2s, ux3s, + phis, weights, tags, pldis, + xi_Cd, dxi_Cd, v, weight, ZERO); + } else { + InjectParticle( + p + offset, + i1s, i2s, i3s, + dx1s, dx2s, dx3s, + ux1s, ux2s, ux3s, + phis, weights, tags, pldis, + xi_Cd, dxi_Cd, v, weight, ZERO, + domain_idx, cntr + p); + } + // clang-format on + } + }; // struct UniformInjectorSingle_kernel + template struct GlobalInjector_kernel { static constexpr auto D = M::Dim; @@ -443,13 +599,13 @@ namespace kernel { if constexpr (M::CoordType != Coord::Cartesian) { phi = in_phi(p); } - if constexpr (S == SimEngine::SRPIC and M::CoordType != Coord::Cartesian) { + if constexpr (::traits::engine::HasImplicitPhiCoordinate) { x_Cd_[2] = phi; } - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { global_metric.template transform_xyz(x_Cd_, u_Ph, u_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { global_metric.template transform(x_Cd, u_Ph, u_Cd); } else { raise::KernelError(HERE, "Unknown simulation engine"); @@ -480,9 +636,9 @@ namespace kernel { vec_t u_Ph { in_ux1(p), in_ux2(p), in_ux3(p) }; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { global_metric.template transform_xyz(x_Cd, u_Ph, u_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { global_metric.template transform(x_Cd, u_Ph, u_Cd); } else { raise::KernelError(HERE, "Unknown simulation engine"); @@ -733,7 +889,7 @@ namespace kernel { coord_t x_Cd_ { ZERO }; x_Cd_[0] = x_Cd[0]; x_Cd_[1] = x_Cd[1]; - if constexpr (S == SimEngine::SRPIC and M::CoordType != Coord::Cartesian) { + if constexpr (::traits::engine::HasImplicitPhiCoordinate) { x_Cd_[2] = ZERO; } metric.template convert(x_Cd, x_Ph); @@ -758,10 +914,12 @@ namespace kernel { { vec_t v_T { ZERO }; energy_dist_1(x_Ph, v_T); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { metric.template transform_xyz(x_Cd_, v_T, v_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { metric.template transform(x_Cd_, v_T, v_Cd); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); } } inject1(index, @@ -773,10 +931,12 @@ namespace kernel { { vec_t v_T { ZERO }; energy_dist_2(x_Ph, v_T); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { metric.template transform_xyz(x_Cd_, v_T, v_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { metric.template transform(x_Cd_, v_T, v_Cd); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); } } inject2(index, @@ -823,10 +983,12 @@ namespace kernel { { vec_t v_T { ZERO }; energy_dist_1(x_Ph, v_T); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { metric.template transform_xyz(x_Cd, v_T, v_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { metric.template transform(x_Cd, v_T, v_Cd); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); } } inject1( @@ -839,10 +1001,12 @@ namespace kernel { { vec_t v_T { ZERO }; energy_dist_2(x_Ph, v_T); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { metric.template transform_xyz(x_Cd, v_T, v_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { metric.template transform(x_Cd, v_T, v_Cd); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); } } inject2( diff --git a/src/kernels/particle_moments.hpp b/src/kernels/particle_moments.hpp index 1244ba21d..dea6bc213 100644 --- a/src/kernels/particle_moments.hpp +++ b/src/kernels/particle_moments.hpp @@ -17,6 +17,7 @@ #include "global.h" #include "arch/kokkos_aliases.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/comparators.h" #include "utils/error.h" @@ -150,7 +151,7 @@ namespace kernel { Inline auto computeStressEnergyComponent(prtlidx_t p) const -> real_t { real_t u0 { ZERO }; vec_t u_Phys { ZERO }; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::StressEnergyInTetradBasis) { // stress-energy tensor for SR is computed in the tetrad (hatted) basis if constexpr (M::CoordType == Coord::Cartesian) { u_Phys[0] = particles.ux1(p); @@ -166,7 +167,7 @@ namespace kernel { if constexpr (D == Dim::_3D) { x_Code[2] = static_cast(particles.i3(p)) + static_cast(particles.dx3(p)); - } else { + } else if constexpr (::traits::engine::HasImplicitPhiCoordinate) { x_Code[2] = particles.phi(p); } metric.template transform_xyz( @@ -177,7 +178,7 @@ namespace kernel { u0 = (mass == ZERO) ? (NORM(u_Phys[0], u_Phys[1], u_Phys[2])) : (math::sqrt(ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2]))); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::StressEnergyInContravariantBasis) { // stress-energy tensor for GR is computed in contravariant basis static_assert(D != Dim::_1D, "GRPIC 1D"); coord_t x_Code { ZERO }; @@ -247,12 +248,16 @@ namespace kernel { { particles.ux1(p), particles.ux2(p), particles.ux3(p) }, u_Phys); } - if (mass == ZERO) { - u0 = NORM(u_Phys[0], u_Phys[1], u_Phys[2]); + if constexpr (S == SimEngine::HYBRID) { + return mass * u_Phys[c1 - 1]; } else { - u0 = math::sqrt(ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2])); + if (mass == ZERO) { + u0 = NORM(u_Phys[0], u_Phys[1], u_Phys[2]); + } else { + u0 = math::sqrt(ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2])); + } + return (mass == ZERO ? ONE : mass) * u_Phys[c1 - 1] / u0; } - return (mass == ZERO ? ONE : mass) * u_Phys[c1 - 1] / u0; } Inline auto computeEckartVelocityFluxComponent(prtlidx_t p) const -> real_t { @@ -295,7 +300,7 @@ namespace kernel { if constexpr (F == FldsID::T) { coeff = computeStressEnergyComponent(p); } else if constexpr (F == FldsID::V) { - if constexpr (S == SimEngine::GRPIC) { + if constexpr (::traits::engine::IsGR) { coeff = computeEckartVelocityFluxComponent(p); } else { coeff = computeBulk3VelocityTimesMass(p); @@ -744,13 +749,13 @@ namespace kernel { return; } real_t en; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { if (is_massive) { en = U2GAMMA(particles.ux1(p), particles.ux2(p), particles.ux3(p)) - ONE; } else { en = NORM(particles.ux1(p), particles.ux2(p), particles.ux3(p)); } - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { coord_t x_Code { ZERO }; x_Code[0] = static_cast(particles.i1(p)) + static_cast(particles.dx1(p)); diff --git a/src/kernels/particle_shapes.hpp b/src/kernels/particle_shapes.hpp index 8e155b795..507e3ded7 100644 --- a/src/kernels/particle_shapes.hpp +++ b/src/kernels/particle_shapes.hpp @@ -541,6 +541,43 @@ namespace prtl_shape { } } + /** + * @brief Spline shape S_O(x) with the order O selected at compile time. + * Compile-time analogue of ParticleMoments_kernel::shapeFunction. + * @tparam O shape order: 0 = NGP (constant), 1 = CIC, 2 = TSC, ... up to 11. + */ + template + Inline auto particle_shape(const real_t x) -> real_t { + if constexpr (O == 0u) { + return S1(x); + } else if constexpr (O == 1u) { + return S1(x); + } else if constexpr (O == 2u) { + return S2(x); + } else if constexpr (O == 3u) { + return S3(x); + } else if constexpr (O == 4u) { + return S4(x); + } else if constexpr (O == 5u) { + return S5(x); + } else if constexpr (O == 6u) { + return S6(x); + } else if constexpr (O == 7u) { + return S7(x); + } else if constexpr (O == 8u) { + return S8(x); + } else if constexpr (O == 9u) { + return S9(x); + } else if constexpr (O == 10u) { + return S10(x); + } else if constexpr (O == 11u) { + return S11(x); + } else { + static_assert(O <= 11u, "particle_shape: unsupported shape order (max 11)"); + return ZERO; + } + } + template Inline void order(const int& i, const real_t& di, int& i_min, real_t S[O + 1]) { if constexpr (O == 1u) { diff --git a/src/kernels/prtls_to_phys.hpp b/src/kernels/prtls_to_phys.hpp index b2f570a5d..eade079bb 100644 --- a/src/kernels/prtls_to_phys.hpp +++ b/src/kernels/prtls_to_phys.hpp @@ -19,6 +19,7 @@ #include "global.h" #include "arch/kokkos_aliases.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/error.h" #include "utils/numeric.h" @@ -108,7 +109,7 @@ namespace kernel { if constexpr ((D == Dim::_2D) || (D == Dim::_3D)) { raise::ErrorIf(buff_x2.extent(0) == 0, "Invalid buffer size", HERE); } - if constexpr (((D == Dim::_2D) && (M::CoordType != Coord::Cartesian)) || + if constexpr (::traits::engine::HasImplicitPhiCoordinate or (D == Dim::_3D)) { raise::ErrorIf(buff_x3.extent(0) == 0, "Invalid buffer size", HERE); } @@ -132,75 +133,54 @@ namespace kernel { } Inline void bufferX(prtlidx_t p_from, prtlidx_t p_to) const { - if constexpr ((D == Dim::_1D) || (D == Dim::_2D) || (D == Dim::_3D)) { + if constexpr (D == Dim::_1D or D == Dim::_2D or D == Dim::_3D) { buff_x1(p_to) = metric.template convert<1, Crd::Cd, Crd::Ph>( static_cast(i1(p_from)) + static_cast(dx1(p_from))); } - if constexpr ((D == Dim::_2D) || (D == Dim::_3D)) { + if constexpr (D == Dim::_2D or D == Dim::_3D) { buff_x2(p_to) = metric.template convert<2, Crd::Cd, Crd::Ph>( static_cast(i2(p_from)) + static_cast(dx2(p_from))); } - if constexpr ((D == Dim::_2D) && (M::CoordType != Coord::Cartesian)) { - buff_x3(p_to) = phi(p_from); - } else if constexpr (D == Dim::_3D) { + if constexpr (D == Dim::_3D) { buff_x3(p_to) = metric.template convert<3, Crd::Cd, Crd::Ph>( static_cast(i3(p_from)) + static_cast(dx3(p_from))); + } else if constexpr (::traits::engine::HasImplicitPhiCoordinate) { + buff_x3(p_to) = phi(p_from); } } Inline void bufferU(prtlidx_t p_from, prtlidx_t p_to) const { - vec_t u_Phys { ZERO }; - if constexpr (D == Dim::_1D) { - if constexpr (M::CoordType == Coord::Cartesian) { - metric.template transform_xyz( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else { - raise::KernelError(HERE, "Unsupported coordinate system in 1D"); - } - } else if constexpr (D == Dim::_2D) { - if constexpr (M::CoordType == Coord::Cartesian) { - metric.template transform_xyz( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)), - static_cast(i2(p_from)) + static_cast(dx2(p_from)) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else if constexpr (S == SimEngine::SRPIC) { - metric.template transform_xyz( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)), - static_cast(i2(p_from)) + static_cast(dx2(p_from)), - phi(p_from) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else if constexpr (S == SimEngine::GRPIC) { - metric.template transform( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)), - static_cast(i2(p_from)) + static_cast(dx2(p_from)) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else { - raise::KernelError(HERE, "Unrecognized simulation engine"); - } - } else if constexpr (D == Dim::_3D) { - if constexpr (S == SimEngine::SRPIC) { - metric.template transform_xyz( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)), - static_cast(i2(p_from)) + static_cast(dx2(p_from)), - static_cast(i3(p_from)) + static_cast(dx3(p_from)) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else if constexpr (S == SimEngine::GRPIC) { - metric.template transform( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)), - static_cast(i2(p_from)) + static_cast(dx2(p_from)), - static_cast(i3(p_from)) + static_cast(dx3(p_from)) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else { - raise::KernelError(HERE, "Unrecognized simulation engine"); - } + vec_t u_Phys { ZERO }; + coord_t x_Code { ZERO }; + if constexpr (D == Dim::_1D or D == Dim::_2D or D == Dim::_3D) { + x_Code[0] = static_cast(i1(p_from)) + + static_cast(dx1(p_from)); + } + if constexpr (D == Dim::_2D or D == Dim::_3D) { + x_Code[1] = static_cast(i2(p_from)) + + static_cast(dx2(p_from)); } + if constexpr (D == Dim::_3D) { + x_Code[2] = static_cast(i3(p_from)) + + static_cast(dx3(p_from)); + } else if constexpr (::traits::engine::HasImplicitPhiCoordinate) { + x_Code[2] = phi(p_from); + } + + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { + metric.template transform_xyz( + x_Code, + { ux1(p_from), ux2(p_from), ux3(p_from) }, + u_Phys); + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { + metric.template transform( + x_Code, + { ux1(p_from), ux2(p_from), ux3(p_from) }, + u_Phys); + } else { + raise::KernelError(HERE, "Unrecognized simulation engine"); + } + buff_ux1(p_to) = u_Phys[0]; buff_ux2(p_to) = u_Phys[1]; buff_ux3(p_to) = u_Phys[2]; diff --git a/src/kernels/pushers/sr_policies.h b/src/kernels/pushers/sr_policies.h index b247b27b7..efcef1c78 100644 --- a/src/kernels/pushers/sr_policies.h +++ b/src/kernels/pushers/sr_policies.h @@ -102,6 +102,62 @@ namespace kernel::sr { } } + // The optional-method dispatch below is split into these free function + // templates on purpose. `if constexpr` whose condition depends only on a + // template parameter captured from an *enclosing* scope is not reliably + // discarded inside a generic lambda: several compilers (nvcc, intel, ...) + // still parse/instantiate the dead branch, which fails when the pgen does + // not define the optional method. Making the predicate a template parameter + // of a regular function template guarantees the dead branch is discarded. + + template + void DispatchEmissionPolicy(const PGen& pgen, + DOM& domain, + const PusherContext& pusher_ctx, + Next&& next) { + if constexpr (HasEP) { + next(pgen.EmissionPolicy(pusher_ctx.time, pusher_ctx.species_index, domain)); + } else { + raise::Error("Custom emission policy flag is set but problem " + "generator does not define an emission policy", + HERE); + } + } + + template + void DispatchCustomPrtlUpdate(const PGen& pgen, + DOM& domain, + const PusherContext& pusher_ctx, + Next&& next) { + if constexpr (HasCPU) { + next(pgen.CustomParticleUpdate(pusher_ctx.time, + pusher_ctx.species_index, + domain)); + } else { + next(::traits::custom_prtl_update::NoPolicy_t {}); + } + } + + template + void DispatchExternalFields(const PGen& pgen, + DOM& domain, + const PusherContext& pusher_ctx, + Next&& next) { + if constexpr (HasEF) { + const auto [apply_extfields, external_fields] = pgen.ExternalFields( + pusher_ctx.time, + pusher_ctx.species_index, + domain); + if (apply_extfields) { + next(external_fields); + } else { + next(::traits::extfields::NoPolicy_t {}); + } + } else { + next(::traits::extfields::NoPolicy_t {}); + } + } + template void MakePusherPolicy(const PGen& pgen, DOM& domain, @@ -110,6 +166,10 @@ namespace kernel::sr { ntt::EmissionTypeFlag emission_type, bool atm, F&& callback) { + constexpr bool has_emission = ::traits::pgen::HasEmissionPolicy; + constexpr bool has_cpu = ::traits::pgen::HasCustomPrtlUpdate; + constexpr bool has_extfields = ::traits::pgen::HasExternalFields; + auto with_emission = [&](auto next) { switch (emission_type) { case ntt::EmissionType::SYNCHROTRON: @@ -125,15 +185,7 @@ namespace kernel::sr { pusher_ctx)); break; case ntt::EmissionType::CUSTOM: - if constexpr (::traits::pgen::HasEmissionPolicy) { - next(pgen.EmissionPolicy(pusher_ctx.time, - pusher_ctx.species_index, - domain)); - } else { - raise::Error("Custom emission policy flag is set but problem " - "generator does not define an emission policy", - HERE); - } + DispatchEmissionPolicy(pgen, domain, pusher_ctx, next); break; case ntt::EmissionType::NONE: default: @@ -143,29 +195,11 @@ namespace kernel::sr { }; auto with_custom_prtl_upd = [&](auto next) { - if constexpr (::traits::pgen::HasCustomPrtlUpdate) { - next(pgen.CustomParticleUpdate(pusher_ctx.time, - pusher_ctx.species_index, - domain)); - } else { - next(::traits::custom_prtl_update::NoPolicy_t {}); - } + DispatchCustomPrtlUpdate(pgen, domain, pusher_ctx, next); }; auto with_ext_fields = [&](auto next) { - if constexpr (::traits::pgen::HasExternalFields) { - const auto [apply_extfields, external_fields] = pgen.ExternalFields( - pusher_ctx.time, - pusher_ctx.species_index, - domain); - if (apply_extfields) { - next(external_fields); - } else { - next(::traits::extfields::NoPolicy_t {}); - } - } else { - next(::traits::extfields::NoPolicy_t {}); - } + DispatchExternalFields(pgen, domain, pusher_ctx, next); }; with_emission([&](auto ep) { diff --git a/src/kernels/reduced_stats.hpp b/src/kernels/reduced_stats.hpp index a6411f681..f80d7abcd 100644 --- a/src/kernels/reduced_stats.hpp +++ b/src/kernels/reduced_stats.hpp @@ -15,6 +15,7 @@ #include "global.h" #include "arch/kokkos_aliases.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/numeric.h" @@ -461,7 +462,7 @@ namespace kernel { // for stress-energy tensor real_t energy { ZERO }; vec_t u_Phys { ZERO }; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::StressEnergyInTetradBasis) { // SR // stress-energy tensor for SR is computed in the tetrad (hatted) basis if constexpr (M::CoordType == Coord::Cartesian) { @@ -478,7 +479,8 @@ namespace kernel { if constexpr (D == Dim::_3D) { x_Code[2] = static_cast(particles.i3(p)) + static_cast(particles.dx3(p)); - } else { + } else if constexpr ( + ::traits::engine::HasImplicitPhiCoordinate) { x_Code[2] = particles.phi(p); } metric.template transform_xyz( @@ -492,7 +494,8 @@ namespace kernel { energy = mass * math::sqrt( ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2])); } - } else { + } else if constexpr ( + ::traits::engine::StressEnergyInContravariantBasis) { // GR // stress-energy tensor for GR is computed in contravariant basis static_assert(D != Dim::_1D, "GRPIC 1D"); @@ -519,6 +522,8 @@ namespace kernel { energy = mass * math::sqrt(ONE + energy); } metric.template transform(x_Code, u_Cntrv, u_Phys); + } else { + raise::KernelError(HERE, "Unsupported engine for stress-energy tensor"); } // compute the corresponding moment real_t coeff = ONE; diff --git a/src/metrics/kerr_schild.h b/src/metrics/kerr_schild.h index 2f225a4f5..7338a0216 100644 --- a/src/metrics/kerr_schild.h +++ b/src/metrics/kerr_schild.h @@ -26,6 +26,7 @@ #include namespace metric { + using namespace ntt; template class KerrSchild : public MetricBase { @@ -59,10 +60,10 @@ namespace metric { } public: - static constexpr const char* Label { "kerr_schild" }; - static constexpr Dimension PrtlDim { D }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Spherical }; - static constexpr ntt::Metric MetricType { ntt::Metric::Kerr_Schild }; + static constexpr const char* Label { "kerr_schild" }; + static constexpr Dimension PrtlDim { D }; + static constexpr Coord::type CoordType { Coord::Spherical }; + static constexpr Metric::type MetricType { Metric::Kerr_Schild }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/metrics/kerr_schild_0.h b/src/metrics/kerr_schild_0.h index fc4d04c9c..e6b3b9de2 100644 --- a/src/metrics/kerr_schild_0.h +++ b/src/metrics/kerr_schild_0.h @@ -27,6 +27,7 @@ #include namespace metric { + using namespace ntt; template class KerrSchild0 : public MetricBase { @@ -39,10 +40,10 @@ namespace metric { const real_t dr_inv, dtheta_inv, dphi_inv; public: - static constexpr const char* Label { "kerr_schild_0" }; - static constexpr Dimension PrtlDim { D }; - static constexpr ntt::Metric MetricType { ntt::Metric::Kerr_Schild_0 }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Spherical }; + static constexpr const char* Label { "kerr_schild_0" }; + static constexpr Dimension PrtlDim { D }; + static constexpr Metric::type MetricType { Metric::Kerr_Schild_0 }; + static constexpr Coord::type CoordType { Coord::Spherical }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/metrics/minkowski.h b/src/metrics/minkowski.h index 3a679b389..166c0624c 100644 --- a/src/metrics/minkowski.h +++ b/src/metrics/minkowski.h @@ -26,16 +26,17 @@ #include namespace metric { + using namespace ntt; template class Minkowski : public MetricBase { const real_t dx, dx_inv; public: - static constexpr const char* Label { "minkowski" }; - static constexpr Dimension PrtlDim { D }; - static constexpr ntt::Metric MetricType { ntt::Metric::Minkowski }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Cartesian }; + static constexpr const char* Label { "minkowski" }; + static constexpr Dimension PrtlDim { D }; + static constexpr Metric::type MetricType { Metric::Minkowski }; + static constexpr Coord::type CoordType { Coord::Cartesian }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index 7b7c3646e..87bea03a4 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -23,6 +23,7 @@ #include "metrics/metric_base.h" namespace metric { + using namespace ntt; template class QKerrSchild : public MetricBase { @@ -58,10 +59,10 @@ namespace metric { } public: - static constexpr const char* Label { "qkerr_schild" }; - static constexpr Dimension PrtlDim { D }; - static constexpr ntt::Metric MetricType { ntt::Metric::QKerr_Schild }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Qspherical }; + static constexpr const char* Label { "qkerr_schild" }; + static constexpr Dimension PrtlDim { D }; + static constexpr Metric::type MetricType { Metric::QKerr_Schild }; + static constexpr Coord::type CoordType { Coord::Qspherical }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/metrics/qspherical.h b/src/metrics/qspherical.h index 652991300..3f1500acf 100644 --- a/src/metrics/qspherical.h +++ b/src/metrics/qspherical.h @@ -28,6 +28,7 @@ #include namespace metric { + using namespace ntt; template class QSpherical : public MetricBase { @@ -41,10 +42,10 @@ namespace metric { const bool small_angle; public: - static constexpr const char* Label { "qspherical" }; - static constexpr Dimension PrtlDim = Dim::_3D; - static constexpr ntt::Metric MetricType { ntt::Metric::QSpherical }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Qspherical }; + static constexpr const char* Label { "qspherical" }; + static constexpr Dimension PrtlDim = Dim::_3D; + static constexpr Metric::type MetricType { Metric::QSpherical }; + static constexpr Coord::type CoordType { Coord::Qspherical }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/metrics/spherical.h b/src/metrics/spherical.h index 72ca69368..aef0df67c 100644 --- a/src/metrics/spherical.h +++ b/src/metrics/spherical.h @@ -25,6 +25,7 @@ #include namespace metric { + using namespace ntt; template class Spherical : public MetricBase { @@ -36,10 +37,10 @@ namespace metric { const bool small_angle { false }; public: - static constexpr const char* Label { "spherical" }; - static constexpr Dimension PrtlDim { Dim::_3D }; - static constexpr ntt::Metric MetricType { ntt::Metric::Spherical }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Spherical }; + static constexpr const char* Label { "spherical" }; + static constexpr Dimension PrtlDim { Dim::_3D }; + static constexpr Metric::type MetricType { Metric::Spherical }; + static constexpr Coord::type CoordType { Coord::Spherical }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/output/fields.cpp b/src/output/fields.cpp index 2fa5b00ce..ffdf7e18d 100644 --- a/src/output/fields.cpp +++ b/src/output/fields.cpp @@ -3,6 +3,7 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" #include "utils/error.h" #include "utils/formatting.h" @@ -46,7 +47,7 @@ namespace out { comp = { { 1 }, { 2 }, { 3 } }; } else if (id() == FldsID::V) { // bulk velocity: 3D for SR, 4D for GR - if (S == SimEngine::SRPIC) { + if (::traits::engine::isSR(S)) { // SR: always 3-velocity components comp = { { 1 }, { 2 }, { 3 } }; } else { @@ -65,7 +66,7 @@ namespace out { } // data preparation flags if (not(is_moment() or is_custom() or is_divergence())) { - if (S == SimEngine::SRPIC) { + if (::traits::engine::userFieldsInTetradBasis(S)) { prepare_flag = PrepareOutput::ConvertToHat; } else { prepare_flag = is_gr_aux_field() ? PrepareOutput::ConvertToPhysCov diff --git a/tests/global/enums.cpp b/tests/global/enums.cpp index 66f1e2274..2a5772852 100644 --- a/tests/global/enums.cpp +++ b/tests/global/enums.cpp @@ -55,7 +55,7 @@ auto main() -> int { enum_str_t all_coords = { "cart", "sph", "qsph" }; enum_str_t all_metrics = { "minkowski", "spherical", "qspherical", "kerr_schild", "qkerr_schild", "kerr_schild_0" }; - enum_str_t all_simulation_engines = { "srpic", "grpic" }; + enum_str_t all_simulation_engines = { "srpic", "grpic", "hybrid" }; enum_str_t all_particle_bcs = { "periodic", "absorb", "atmosphere", "custom", "reflect", "horizon", "axis", "sync" }; enum_str_t all_fields_bcs = { "periodic", "match", "fixed", diff --git a/tests/kernels/particle_moments.cpp b/tests/kernels/particle_moments.cpp index cbda7546f..274280f74 100644 --- a/tests/kernels/particle_moments.cpp +++ b/tests/kernels/particle_moments.cpp @@ -79,9 +79,6 @@ void testParticleMoments(const std::vector& res, EmissionType::NONE, 0, 0 }; - - const float mass = 1.0; - const float charge = 1.0; const bool use_weights = false; const real_t inv_n0 = 1.0;