From 703537c793e823eacc07cf601f3f2d64aa28dc6d Mon Sep 17 00:00:00 2001 From: Neco Kriel Date: Wed, 8 May 2024 13:44:31 +1000 Subject: [PATCH 01/37] add call to solve induction eqn --- src/RadhydroSimulation.hpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 10e684acf..ccb7095ec 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -54,6 +54,7 @@ #include "TabulatedCooling.hpp" #include "eos.H" #include "hydro_system.hpp" +#include "mhd_system.hpp" #include "hyperbolic_system.hpp" #include "physics_info.hpp" #include "physics_numVars.hpp" @@ -72,6 +73,7 @@ template class RadhydroSimulation : public AMRSimulation::TracerPC; using AMRSimulation::nghost_cc_; + using AMRSimulation::nghost_fc_; using AMRSimulation::areInitialConditionsDefined_; using AMRSimulation::BCs_cc_; using AMRSimulation::BCs_fc_; @@ -123,6 +125,7 @@ template class RadhydroSimulation : public AMRSimulation::nvar_; // hydro static constexpr int ncompHyperbolic_ = RadSystem::nvarHyperbolic_; static constexpr int nstartHyperbolic_ = RadSystem::nstartHyperbolic_; + static constexpr int n_mhd_vars_per_dim_ = MHDSystem::nvar_per_dim_; // mhd amrex::Real radiationCflNumber_ = 0.3; int maxSubsteps_ = 10; // maximum number of radiation subcycles per hydro step @@ -1100,6 +1103,14 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::iMultiFab redoFlag(grids[lev], dmap[lev], 1, 1); redoFlag.setVal(quokka::redoFlag::none); + if constexpr (Physics_Traits::is_mhd_enabled) { + std::array rhs_fc; + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto ba_fc = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); + rhs_fc[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, 0); + } + MHDSystem::ComputeEMF(rhs_fc, stateOld, stateOld, dx, nghost_cc_, nghost_fc_); + } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); From d79c692b80a57c97f281112af79e93744750fede Mon Sep 17 00:00:00 2001 From: Neco Kriel Date: Wed, 8 May 2024 13:44:59 +1000 Subject: [PATCH 02/37] draft func to solve induction eqn --- src/mhd_system.hpp | 296 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 295 insertions(+), 1 deletion(-) diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index 4a320ece9..74c90f89e 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -14,10 +14,12 @@ // internal headers #include "physics_info.hpp" +#include "hydro_system.hpp" +#include "hyperbolic_system.hpp" #include "physics_numVars.hpp" /// Class for a MHD system of conservation laws -template class MHDSystem +template class MHDSystem : public HyperbolicSystem { public: static constexpr int nvar_per_dim_ = Physics_NumVars::numMHDVars_per_dim; @@ -26,6 +28,298 @@ template class MHDSystem enum varIndex_perDim { bfield_index = Physics_Indices::mhdFirstIndex, }; + + static void ComputeEMF(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, amrex::GpuArray dx, int nghost_cc, int nghost_fc); + + static void ReconstructTo(FluxDir dir, amrex::FArrayBox const &q_fab, amrex::FArrayBox &leftState_fab, amrex::FArrayBox &rightState_fab, const amrex::Box &indexRange, int reconstructionOrder, int nghost); }; +template +void MHDSystem::ComputeEMF(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, amrex::GpuArray dx, const int nghost_cc, const int nghost_fc) +{ + std::cout << "Computing EMF" << std::endl; + + // Loop over each box-array on the level + // Note: all the different centerings still have the same distribution mapping, so it is fine for us to attach our looping to cc FArrayBox + for (amrex::MFIter mfi(cc_mf_cVars); mfi.isValid(); ++mfi) { + const amrex::Box &box_cc = mfi.validbox(); + + // In this function we distinguish between world (w:3), array (i:2), quandrant (q:4), and component (x:3) indexing with prefixes. We will use the x-prefix when the w- and i- indexes are the same. + // We will minimise the storage footprint by only computing and holding onto the quantities required for calculating the EMF in the w-direction. This inadvertently leads to duplicate computation, but also significantly reduces the memory footprint, which is a bigger bottleneck. + + // initialise the rhs of the induction equation + for (int windex = 0; windex < AMREX_SPACEDIM; ++windex) { + const auto &fcxw_a4_rhs = fcx_mf_rhs[windex][mfi].array(); + amrex::ParallelFor(box_cc, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + fcxw_a4_rhs(i, j, k, MHDSystem::bfield_index) = 0; + }); + } + + // extract cell-centered velocity fields + // indexing: field[3: x-component] + std::array cc_fabs_Ux; + cc_fabs_Ux[0].resize(box_cc, 1); + cc_fabs_Ux[1].resize(box_cc, 1); + cc_fabs_Ux[2].resize(box_cc, 1); + const auto &cc_a4_Ux0 = cc_fabs_Ux[0].array(); + const auto &cc_a4_Ux1 = cc_fabs_Ux[1].array(); + const auto &cc_a4_Ux2 = cc_fabs_Ux[2].array(); + const auto &cc_a4_cVars = cc_mf_cVars[mfi].const_array(); + amrex::ParallelFor(box_cc, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + const auto rho = cc_a4_cVars(i, j, k, HydroSystem::density_index); + const auto px1 = cc_a4_cVars(i, j, k, HydroSystem::x1Momentum_index); + const auto px2 = cc_a4_cVars(i, j, k, HydroSystem::x2Momentum_index); + const auto px3 = cc_a4_cVars(i, j, k, HydroSystem::x3Momentum_index); + cc_a4_Ux0(i,j,k) = px1 / rho; + cc_a4_Ux1(i,j,k) = px2 / rho; + cc_a4_Ux2(i,j,k) = px3 / rho; + }); + + // // indexing: field[3: x-component/x-face] + // // create a view of the b-field data (do not make another copy) + // std::array fc_fabs_Bx = { + // FArrayBox(fcx_mf_cVars[0][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + // FArrayBox(fcx_mf_cVars[1][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + // FArrayBox(fcx_mf_cVars[2][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + // }; + + // alternatively, make a complete copy of the b-field data + std::array fc_fabs_Bx; + for (int w_index = 0; w_index < 3; ++w_index) { + const amrex::Box box_fc = amrex::convert(box_cc, amrex::IntVect::TheDimensionVector(w_index)); + fc_fabs_Bx[w_index].resize(box_fc, 1); + const auto &fc_a4_Bx = fc_fabs_Bx[w_index].array(); + // extract face-centered magnetic fields + const auto &fc_a4_cVars = fcx_mf_cVars[w_index][mfi].const_array(); + amrex::ParallelFor(box_fc, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + fc_a4_Bx(i, j, k) = fc_a4_cVars(i, j, k, MHDSystem::bfield_index); + }); + } + + + // compute the magnetic flux through each cell-face + for (int wsolve = 0; wsolve < 3; ++wsolve) { + // electric field on the cell-edge + // indexing: field[2: i-edge on cell-face] + std::array eci_fabs_E; + // to solve for the magnetic flux through each face we compute the line integral of the EMF around the cell-face. + // so let's solve for the EMF along each of the two edges along the edge of the cell-face + for (int iedge_rel2face = 0; iedge_rel2face < 2; ++iedge_rel2face) { + // for each of the two cell-edges on the cell-face + + // define the two directions we need to extrapolate cell-centered velocity fields to get them to the cell-edge + // we will want to compute E2 = (U0 * B1 - U1 * B0) along the cell-edge + std::array w_extrap_dirs = { + wsolve, // dir-0: w-direction of the cell-face being solved for (relative to the cell-center) + (wsolve + iedge_rel2face + 1) % 3 // dir-1: w-direction of the cell-edge being solved for (relative to the cell-face) + }; + const amrex::Box box_ec = amrex::convert(box_cc, amrex::IntVect::TheDimensionVector(w_extrap_dirs[0]) + amrex::IntVect::TheDimensionVector(w_extrap_dirs[1])); + + // define output electric field on the cell-edge + eci_fabs_E[iedge_rel2face].resize(box_ec, 1); + + // initialise FArrayBox for storing the edge-centered velocity fields averaged across the two extrapolation permutations + // indexing: field[2: i-compnent][4: quadrant around edge] + std::array, 2> ec_fabs_Ui_q; + // initialise temporary FArrayBox for storing the edge-centered velocity fields reconstructed from the cell-face + // indexing: field[2: i-side of edge] + std::array ec_fabs_U_ieside; + // define the four possible velocity field quantities that could be reconstructed at the cell-edge + // also define the temporary velocity field quantities that will be used for computing the extrapolation + ec_fabs_U_ieside[0].resize(box_ec, 1); + ec_fabs_U_ieside[1].resize(box_ec, 1); + // indexing: field[2: i-compnent][2: i-side of edge] + // note: magnetic field components cannot be discontinuous along themselves (i.e., either side of the face where they are stored) + std::array, 2> ec_fabs_Bi_ieside; + // define quantities + for (int icomp = 0; icomp < 2; ++icomp) { + ec_fabs_Bi_ieside[icomp][0].resize(box_ec, 1); + ec_fabs_Bi_ieside[icomp][1].resize(box_ec, 1); + for (int iquad = 0; iquad < 4; ++iquad) { + ec_fabs_Ui_q[icomp][iquad].resize(box_ec, 1); + } + } + + // extrapolate the two required cell-centered velocity field components to the cell-edge + // there are two possible permutations for doing this, that is getting cell-centered quanties to a cell-edge + // first is cc->fc[dir-0]->ec and second is cc->fc[dir-1]->ec + for (int iperm = 0; iperm < 2; ++iperm) { + // for each permutation of extrapolating cc->ec + + // define quantities required for creating face-centered FArrayBox + const int w_extrap_dir2face = w_extrap_dirs[iperm]; + const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(w_extrap_dir2face); + const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); + const auto dir2face = static_cast(w_extrap_dir2face); + + // define extrapolation direction to go from face to edge + const int w_extrap_dir2edge = w_extrap_dirs[(iperm+1) % 2]; + const auto dir2edge = static_cast(w_extrap_dir2edge); + + // create temporary FArrayBox for storing the face-centered velocity fields reconstructed from the cell-center + // indexing: field[2: i-compnent][2: i-side of face] + std::array, 2> fc_fabs_Ui_ifside; + // extrapolate both required cell-centered velocity fields to the cell-edge + for (int icomp = 0; icomp < 2; ++icomp) { + const int w_comp = w_extrap_dirs[icomp]; + fc_fabs_Ui_ifside[icomp][0].resize(box_fc, 1); + fc_fabs_Ui_ifside[icomp][1].resize(box_fc, 1); + // extrapolate cell-centered velocity components to the cell-face + MHDSystem::ReconstructTo(dir2face, cc_fabs_Ux[w_comp], fc_fabs_Ui_ifside[icomp][0], fc_fabs_Ui_ifside[icomp][1], box_cc, 1, nghost_cc); + // extrapolate face-centered velocity components to the cell-edge + for (int iface = 0; iface < 2; ++iface) { + // reset values in temporary FArrayBox + ec_fabs_U_ieside[0].setVal(0.0); + ec_fabs_U_ieside[1].setVal(0.0); + // extrapolate face-centered velocity component to the cell-edge + MHDSystem::ReconstructTo(dir2edge, fc_fabs_Ui_ifside[icomp][iface], ec_fabs_U_ieside[0], ec_fabs_U_ieside[1], box_fc, 1, nghost_cc); + // figure out which quadrant of the cell-edge this extrapolated velocity component corresponds with + int iquad0 = -1; + int iquad1 = -1; + // note: quadrants are defined based on where the quantity sits relative to the edge (dir-0, dir-1): + // (-,+) | (+,+) + // 1 | 2 + // ------+------ + // 0 | 3 + // (-,-) | (+,-) + if (iperm == 0) { + iquad0 = (iface == 0) ? 0 : 3; + iquad1 = (iface == 0) ? 1 : 2; + } else { + iquad0 = (iface == 0) ? 0 : 1; + iquad1 = (iface == 0) ? 3 : 2; + } + ec_fabs_Ui_q[icomp][iquad0].atomicAdd(ec_fabs_U_ieside[0], 0, 0, 1); + ec_fabs_Ui_q[icomp][iquad1].atomicAdd(ec_fabs_U_ieside[1], 0, 0, 1); + } + } + } + // finish averaging the two different ways for extrapolating cc->ec velocity fields + for (int icomp = 0; icomp < 2; ++icomp) { + for (int iquad = 0; iquad < 4; ++iquad) { + ec_fabs_Ui_q[icomp][iquad].mult(0.5, 0, 1); + } + } + + // extrapolate the two required face-centered magnetic field components to the cell-edge + for (int icomp = 0; icomp < 2; ++icomp) { + // define extrapolation direction to go from face to edge + const int w_comp = w_extrap_dirs[icomp]; + const int w_extrap_dir2edge = w_extrap_dirs[(icomp+1) % 2]; + const auto dir2edge = static_cast(w_extrap_dir2edge); + const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(w_extrap_dir2edge); + const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); + // extrapolate face-centered magnetic components to the cell-edge + MHDSystem::ReconstructTo(dir2edge, fc_fabs_Bx[w_comp], ec_fabs_Bi_ieside[icomp][0], ec_fabs_Bi_ieside[icomp][1], box_fc, 1, nghost_fc); + } + + // indexing: field[4: quadrant around edge] + std::array ec_fabs_E_q; + // compute the EMF along the cell-edge + for (int iquad = 0; iquad < 4; ++iquad) { + // define EMF FArrayBox + ec_fabs_E_q[iquad].resize(box_ec, 1); + // extract relevant velocity and magnetic field components + const auto &U0_qi = ec_fabs_Ui_q[0][iquad].const_array(); + const auto &U1_qi = ec_fabs_Ui_q[1][iquad].const_array(); + const auto &B0_qi = ec_fabs_Bi_ieside[0][(iquad == 0 || iquad == 3) ? 0 : 1].const_array(); + const auto &B1_qi = ec_fabs_Bi_ieside[1][(iquad < 2) ? 0 : 1].const_array(); + // compute electric field in the quadrant about the cell-edge: cross product between velocity and magnetic field in that quadrant + const auto &E2_qi = ec_fabs_E_q[iquad].array(); + amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + E2_qi(i,j,k) = U0_qi(i,j,k) * B1_qi(i,j,k) - U1_qi(i,j,k) * B0_qi(i,j,k); + }); + } + + // // extract both components of magnetic field either side of the cell-edge + // const auto &B0_m = ec_fabs_Bi_ieside[0][0].const_array(); + // const auto &B0_p = ec_fabs_Bi_ieside[0][1].const_array(); + // const auto &B1_m = ec_fabs_Bi_ieside[1][0].const_array(); + // const auto &B1_p = ec_fabs_Bi_ieside[1][1].const_array(); + // // extract all four quadrants of the electric field about the cell-edge + // const auto &E2_q0 = ec_fabs_E_q[0].const_array(); + // const auto &E2_q1 = ec_fabs_E_q[1].const_array(); + // const auto &E2_q2 = ec_fabs_E_q[2].const_array(); + // const auto &E2_q3 = ec_fabs_E_q[3].const_array(); + // // compute electric field on the cell-edge + // const auto &E2_ave = eci_fabs_E[iedge_rel2face].array(); + // amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + // const double spd_x0_m = 1.0; + // const double spd_x0_p = 1.0; + // const double spd_x1_m = 1.0; + // const double spd_x1_p = 1.0; + // E2_ave(i,j,k) = \ + // spd_x0_p * spd_x1_p * E2_q0(i,j,k) + + // spd_x0_m * spd_x1_p * E2_q3(i,j,k) + + // spd_x0_m * spd_x1_m * E2_q1(i,j,k) + + // spd_x0_p * spd_x1_m * E2_q2(i,j,k) - + // spd_x1_p * spd_x1_m / (spd_x1_p + spd_x1_m) * (B0_p(i,j,k) - B0_m(i,j,k)) + + // spd_x0_p * spd_x0_m / (spd_x0_p + spd_x0_m) * (B1_p(i,j,k) - B1_m(i,j,k)); + // }); + + // const int index_E2comp = MHDSystem::bfield_index + (wsolve + 2*iedge_rel2face + 2) % 3; + // std::array idx = {0, 0, 0}; + // idx[index_E2comp] = 1; + // // compute the magnetic flux + // const auto &fcxw_a4_rhs = fcx_mf_rhs[wsolve][mfi].array(); + // amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + // // the induction equation is written in an additive form: the RHS is evaluated in parts as each edge-centered electric field is computed + // fcxw_a4_rhs(i,j,k,MHDSystem::bfield_index) += (E2_ave(i,j,k) - E2_ave(i-idx[0],j-idx[1],k-idx[2])) / dx[wsolve]; + // }); + } + } + } + + std::cout << "Done computing EMF" << std::endl; +} + +template +void MHDSystem::ReconstructTo(FluxDir dir, amrex::FArrayBox const &q_fab, amrex::FArrayBox &leftState_fab, amrex::FArrayBox &rightState_fab, const amrex::Box &indexRange, const int reconstructionOrder, const int nghost) +{ + // N.B.: A one-zone layer around the cells must be fully reconstructed in order for PPM to work. + amrex::Box const &reconstructRange = amrex::grow(indexRange, 1); + amrex::Box const &x1ReconstructRange = amrex::surroundingNodes(reconstructRange, static_cast(dir)); + + if (reconstructionOrder == 3) { + switch (dir) { + case FluxDir::X1: + MHDSystem::template ReconstructStatesPPM(q_fab, leftState_fab, rightState_fab, reconstructRange, 1); + break; + case FluxDir::X2: + MHDSystem::template ReconstructStatesPPM(q_fab, leftState_fab, rightState_fab, reconstructRange, 1); + break; + case FluxDir::X3: + MHDSystem::template ReconstructStatesPPM(q_fab, leftState_fab, rightState_fab, reconstructRange, 1); + break; + } + } else if (reconstructionOrder == 2) { + switch (dir) { + case FluxDir::X1: + MHDSystem::template ReconstructStatesPLM(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); + break; + case FluxDir::X2: + MHDSystem::template ReconstructStatesPLM(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); + break; + case FluxDir::X3: + MHDSystem::template ReconstructStatesPLM(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); + break; + } + } else if (reconstructionOrder == 1) { + switch (dir) { + case FluxDir::X1: + MHDSystem::template ReconstructStatesConstant(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); + break; + case FluxDir::X2: + MHDSystem::template ReconstructStatesConstant(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); + break; + case FluxDir::X3: + MHDSystem::template ReconstructStatesConstant(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); + break; + } + } else { + amrex::Abort("Invalid reconstruction order specified!"); + } +} + + #endif // HYDRO_SYSTEM_HPP_ From ae7a364bc32e7f0c3a54c23eb4f87cdae193990b Mon Sep 17 00:00:00 2001 From: Neco Kriel Date: Wed, 8 May 2024 14:07:42 +1000 Subject: [PATCH 03/37] indicate box-array centering --- src/RadhydroSimulation.hpp | 56 +++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index ccb7095ec..81ccae424 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1034,6 +1034,8 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o fluxScaleFactor = 1.0; } + auto ba_cc = grids[lev]; + auto dm = dmap[lev]; auto dx = geom[lev].CellSizeArray(); // do Strang split source terms (first half-step) @@ -1047,20 +1049,26 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // create temporary multifab for intermediate state amrex::MultiFab state_inter_cc_(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); state_inter_cc_.setVal(0); // prevent assert in fillBoundaryConditions when radiation is enabled + std::array state_inter_fc_; + if constexpr (Physics_Traits::is_mhd_enabled) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); + state_inter_fc_[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, nghost_fc_); + state_inter_fc_[idim].setVal(0); // prevent assert in fillBoundaryConditions when radiation is enabled + } + } // create temporary multifabs for combined RK2 flux and time-average face velocity std::array flux_rk2; std::array avgFaceVel; const int nghost_vel = 2; // 2 ghost faces are needed for tracer particles - auto ba = grids[lev]; - auto dm = dmap[lev]; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); + auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); // initialize flux MultiFab - flux_rk2[idim] = amrex::MultiFab(ba_face, dm, ncompHydro_, 0); + flux_rk2[idim] = amrex::MultiFab(ba_fc, dm, ncompHydro_, 0); flux_rk2[idim].setVal(0); // initialize velocity MultiFab - avgFaceVel[idim] = amrex::MultiFab(ba_face, dm, 1, nghost_vel); + avgFaceVel[idim] = amrex::MultiFab(ba_fc, dm, 1, nghost_vel); avgFaceVel[idim].setVal(0); } @@ -1092,6 +1100,10 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // advance all grids on local processor (Stage 1 of integrator) auto const &stateOld = state_old_cc_tmp; auto &stateNew = state_inter_cc_; + + auto const &stateOld_fc = state_old_fc_tmp; + auto &stateNew_fc = state_inter_fc_; + auto [fluxArrays, faceVel] = computeHydroFluxes(stateOld, ncompHydro_, lev); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -1106,10 +1118,10 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o if constexpr (Physics_Traits::is_mhd_enabled) { std::array rhs_fc; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_fc = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); + auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); rhs_fc[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, 0); } - MHDSystem::ComputeEMF(rhs_fc, stateOld, stateOld, dx, nghost_cc_, nghost_fc_); + MHDSystem::ComputeEMF(rhs_fc, stateOld, stateOld_fc, dx, nghost_cc_, nghost_fc_); } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); @@ -1403,13 +1415,13 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co { BL_PROFILE("RadhydroSimulation::computeHydroFluxes()"); - auto ba = grids[lev]; + auto ba_cc = grids[lev]; auto dm = dmap[lev]; const int flatteningGhost = 2; const int reconstructGhost = 1; // allocate temporary MultiFabs - amrex::MultiFab primVar(ba, dm, nvars, nghost_cc_); + amrex::MultiFab primVar(ba_cc, dm, nvars, nghost_cc_); std::array flatCoefs; std::array flux; std::array facevel; @@ -1417,15 +1429,15 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co std::array rightState; for (int idim = 0; idim < 3; ++idim) { - flatCoefs[idim] = amrex::MultiFab(ba, dm, 1, flatteningGhost); + flatCoefs[idim] = amrex::MultiFab(ba_cc, dm, 1, flatteningGhost); } for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); - leftState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructGhost); - rightState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructGhost); - flux[idim] = amrex::MultiFab(ba_face, dm, nvars, 0); - facevel[idim] = amrex::MultiFab(ba_face, dm, 1, 0); + auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); + leftState[idim] = amrex::MultiFab(ba_fc, dm, nvars, reconstructGhost); + rightState[idim] = amrex::MultiFab(ba_fc, dm, nvars, reconstructGhost); + flux[idim] = amrex::MultiFab(ba_fc, dm, nvars, 0); + facevel[idim] = amrex::MultiFab(ba_fc, dm, 1, 0); } // conserved to primitive variables @@ -1519,23 +1531,23 @@ auto RadhydroSimulation::computeFOHydroFluxes(amrex::MultiFab const & { BL_PROFILE("RadhydroSimulation::computeFOHydroFluxes()"); - auto ba = grids[lev]; + auto ba_cc = grids[lev]; auto dm = dmap[lev]; const int reconstructRange = 1; // allocate temporary MultiFabs - amrex::MultiFab primVar(ba, dm, nvars, nghost_cc_); + amrex::MultiFab primVar(ba_cc, dm, nvars, nghost_cc_); std::array flux; std::array facevel; std::array leftState; std::array rightState; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); - leftState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructRange); - rightState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructRange); - flux[idim] = amrex::MultiFab(ba_face, dm, nvars, 0); - facevel[idim] = amrex::MultiFab(ba_face, dm, 1, 0); + auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); + leftState[idim] = amrex::MultiFab(ba_fc, dm, nvars, reconstructRange); + rightState[idim] = amrex::MultiFab(ba_fc, dm, nvars, reconstructRange); + flux[idim] = amrex::MultiFab(ba_fc, dm, nvars, 0); + facevel[idim] = amrex::MultiFab(ba_fc, dm, 1, 0); } // conserved to primitive variables From a8fe9fd35cceac7f8336dfb1a48a4a044c0c486c Mon Sep 17 00:00:00 2001 From: Neco Kriel Date: Wed, 8 May 2024 16:58:31 +1000 Subject: [PATCH 04/37] initialise fc state variables --- src/RadhydroSimulation.hpp | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 81ccae424..72cf90c76 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -229,7 +229,7 @@ template class RadhydroSimulation : public AMRSimulation &state_old_fc_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, int lev, amrex::Real time, amrex::Real dt_lev) -> bool; void addStrangSplitSources(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev); @@ -927,19 +927,33 @@ void RadhydroSimulation::advanceHydroAtLevelWithRetries(int lev, amre #endif } - // create temporary multifab for old state + // create temporary multifab for old cell-cenetered state amrex::MultiFab state_old_cc_tmp(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); amrex::Copy(state_old_cc_tmp, state_old_cc_[lev], 0, 0, Physics_Indices::nvarTotal_cc, nghost_cc_); + // create temporary array (different faces) of multifabs for old face-centered state + std::array state_old_fc_tmp; + if constexpr (Physics_Traits::is_mhd_enabled) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + state_old_fc_tmp[idim].define(amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), dmap[lev], Physics_Indices::nvarPerDim_fc, nghost_fc_); + amrex::Copy(state_old_fc_tmp[idim], state_old_fc_[lev][idim], 0, 0, Physics_Indices::nvarPerDim_fc, nghost_fc_); + } + } + // subcycle advanceHydroAtLevel, checking return value for (int substep = 0; substep < nsubsteps; ++substep) { if (substep > 0) { // since we are starting a new substep, we need to copy hydro state from // the new state vector to old state vector amrex::Copy(state_old_cc_tmp, state_new_cc_[lev], 0, 0, ncompHydro_, nghost_cc_); + if constexpr (Physics_Traits::is_mhd_enabled) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + amrex::Copy(state_old_fc_tmp[idim], state_new_fc_[lev][idim], 0, 0, Physics_Indices::nvarPerDim_fc, nghost_fc_); + } + } } - success = advanceHydroAtLevel(state_old_cc_tmp, fr_as_crse, fr_as_fine, lev, time, dt_step); + success = advanceHydroAtLevel(state_old_cc_tmp, state_old_fc_tmp, fr_as_crse, fr_as_fine, lev, time, dt_step); if (!success) { if (Verbose()) { @@ -1022,7 +1036,7 @@ template void RadhydroSimulation::printCoordinat } template -auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_old_cc_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, +auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_old_cc_tmp, std::array &state_old_fc_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, int lev, amrex::Real time, amrex::Real dt_lev) -> bool { BL_PROFILE("RadhydroSimulation::advanceHydroAtLevel()"); @@ -1102,7 +1116,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto &stateNew = state_inter_cc_; auto const &stateOld_fc = state_old_fc_tmp; - auto &stateNew_fc = state_inter_fc_; + // auto &stateNew_fc = state_inter_fc_; auto [fluxArrays, faceVel] = computeHydroFluxes(stateOld, ncompHydro_, lev); From 44e1dedec0537eab4c51ea46d2a046a08bcfb5a8 Mon Sep 17 00:00:00 2001 From: Neco Kriel Date: Wed, 8 May 2024 16:59:05 +1000 Subject: [PATCH 05/37] focus on setup to reconstruct --- src/mhd_system.hpp | 215 ++++++++++++++++++++++++--------------------- 1 file changed, 114 insertions(+), 101 deletions(-) diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index 74c90f89e..0dd901447 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -31,7 +31,7 @@ template class MHDSystem : public HyperbolicSystem &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, amrex::GpuArray dx, int nghost_cc, int nghost_fc); - static void ReconstructTo(FluxDir dir, amrex::FArrayBox const &q_fab, amrex::FArrayBox &leftState_fab, amrex::FArrayBox &rightState_fab, const amrex::Box &indexRange, int reconstructionOrder, int nghost); + static void ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &indexRange, int reconstructionOrder, int nghost); }; template @@ -58,9 +58,9 @@ void MHDSystem::ComputeEMF(std::array cc_fabs_Ux; - cc_fabs_Ux[0].resize(box_cc, 1); - cc_fabs_Ux[1].resize(box_cc, 1); - cc_fabs_Ux[2].resize(box_cc, 1); + cc_fabs_Ux[0].resize(amrex::grow(box_cc, nghost_cc), 1); + cc_fabs_Ux[1].resize(amrex::grow(box_cc, nghost_cc), 1); + cc_fabs_Ux[2].resize(amrex::grow(box_cc, nghost_cc), 1); const auto &cc_a4_Ux0 = cc_fabs_Ux[0].array(); const auto &cc_a4_Ux1 = cc_fabs_Ux[1].array(); const auto &cc_a4_Ux2 = cc_fabs_Ux[2].array(); @@ -82,7 +82,6 @@ void MHDSystem::ComputeEMF(std::array::bfield_index, 1), // FArrayBox(fcx_mf_cVars[2][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), // }; - // alternatively, make a complete copy of the b-field data std::array fc_fabs_Bx; for (int w_index = 0; w_index < 3; ++w_index) { @@ -149,7 +148,7 @@ void MHDSystem::ComputeEMF(std::array(w_extrap_dir2face); // define extrapolation direction to go from face to edge @@ -164,72 +163,86 @@ void MHDSystem::ComputeEMF(std::array::ReconstructTo(dir2face, cc_fabs_Ux[w_comp], fc_fabs_Ui_ifside[icomp][0], fc_fabs_Ui_ifside[icomp][1], box_cc, 1, nghost_cc); - // extrapolate face-centered velocity components to the cell-edge - for (int iface = 0; iface < 2; ++iface) { - // reset values in temporary FArrayBox - ec_fabs_U_ieside[0].setVal(0.0); - ec_fabs_U_ieside[1].setVal(0.0); - // extrapolate face-centered velocity component to the cell-edge - MHDSystem::ReconstructTo(dir2edge, fc_fabs_Ui_ifside[icomp][iface], ec_fabs_U_ieside[0], ec_fabs_U_ieside[1], box_fc, 1, nghost_cc); - // figure out which quadrant of the cell-edge this extrapolated velocity component corresponds with - int iquad0 = -1; - int iquad1 = -1; - // note: quadrants are defined based on where the quantity sits relative to the edge (dir-0, dir-1): - // (-,+) | (+,+) - // 1 | 2 - // ------+------ - // 0 | 3 - // (-,-) | (+,-) - if (iperm == 0) { - iquad0 = (iface == 0) ? 0 : 3; - iquad1 = (iface == 0) ? 1 : 2; - } else { - iquad0 = (iface == 0) ? 0 : 1; - iquad1 = (iface == 0) ? 3 : 2; - } - ec_fabs_Ui_q[icomp][iquad0].atomicAdd(ec_fabs_U_ieside[0], 0, 0, 1); - ec_fabs_Ui_q[icomp][iquad1].atomicAdd(ec_fabs_U_ieside[1], 0, 0, 1); - } - } - } - // finish averaging the two different ways for extrapolating cc->ec velocity fields - for (int icomp = 0; icomp < 2; ++icomp) { - for (int iquad = 0; iquad < 4; ++iquad) { - ec_fabs_Ui_q[icomp][iquad].mult(0.5, 0, 1); + // // extrapolate cell-centered velocity components to the cell-face + // switch (dir2face) { + // case FluxDir::X1: + // std::cout << "x1" << std::endl; + // MHDSystem::template ReconstructStatesConstant(cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), fc_fabs_Ui_ifside[icomp][1].array(), box_fc, 1); + // break; + // case FluxDir::X2: + // std::cout << "x2" << std::endl; + // MHDSystem::template ReconstructStatesConstant(cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), fc_fabs_Ui_ifside[icomp][1].array(), box_fc, 1); + // break; + // case FluxDir::X3: + // std::cout << "x3" << std::endl; + // MHDSystem::template ReconstructStatesConstant(cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), fc_fabs_Ui_ifside[icomp][1].array(), box_fc, 1); + // break; + // } + // MHDSystem::ReconstructTo(dir2face, cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), fc_fabs_Ui_ifside[icomp][1].array(), box_cc, 1, nghost_cc); + // // extrapolate face-centered velocity components to the cell-edge + // for (int iface = 0; iface < 2; ++iface) { + // // reset values in temporary FArrayBox + // ec_fabs_U_ieside[0].setVal(0.0); + // ec_fabs_U_ieside[1].setVal(0.0); + // // extrapolate face-centered velocity component to the cell-edge + // MHDSystem::ReconstructTo(dir2edge, fc_fabs_Ui_ifside[icomp][iface], ec_fabs_U_ieside[0], ec_fabs_U_ieside[1], box_fc, 1, nghost_cc); + // // figure out which quadrant of the cell-edge this extrapolated velocity component corresponds with + // int iquad0 = -1; + // int iquad1 = -1; + // // note: quadrants are defined based on where the quantity sits relative to the edge (dir-0, dir-1): + // // (-,+) | (+,+) + // // 1 | 2 + // // ------+------ + // // 0 | 3 + // // (-,-) | (+,-) + // if (iperm == 0) { + // iquad0 = (iface == 0) ? 0 : 3; + // iquad1 = (iface == 0) ? 1 : 2; + // } else { + // iquad0 = (iface == 0) ? 0 : 1; + // iquad1 = (iface == 0) ? 3 : 2; + // } + // ec_fabs_Ui_q[icomp][iquad0].atomicAdd(ec_fabs_U_ieside[0], 0, 0, 1); + // ec_fabs_Ui_q[icomp][iquad1].atomicAdd(ec_fabs_U_ieside[1], 0, 0, 1); + // } } } + // // finish averaging the two different ways for extrapolating cc->ec velocity fields + // for (int icomp = 0; icomp < 2; ++icomp) { + // for (int iquad = 0; iquad < 4; ++iquad) { + // ec_fabs_Ui_q[icomp][iquad].mult(0.5, 0, 1); + // } + // } - // extrapolate the two required face-centered magnetic field components to the cell-edge - for (int icomp = 0; icomp < 2; ++icomp) { - // define extrapolation direction to go from face to edge - const int w_comp = w_extrap_dirs[icomp]; - const int w_extrap_dir2edge = w_extrap_dirs[(icomp+1) % 2]; - const auto dir2edge = static_cast(w_extrap_dir2edge); - const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(w_extrap_dir2edge); - const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); - // extrapolate face-centered magnetic components to the cell-edge - MHDSystem::ReconstructTo(dir2edge, fc_fabs_Bx[w_comp], ec_fabs_Bi_ieside[icomp][0], ec_fabs_Bi_ieside[icomp][1], box_fc, 1, nghost_fc); - } + // // extrapolate the two required face-centered magnetic field components to the cell-edge + // for (int icomp = 0; icomp < 2; ++icomp) { + // // define extrapolation direction to go from face to edge + // const int w_comp = w_extrap_dirs[icomp]; + // const int w_extrap_dir2edge = w_extrap_dirs[(icomp+1) % 2]; + // const auto dir2edge = static_cast(w_extrap_dir2edge); + // const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(w_extrap_dir2edge); + // const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); + // // extrapolate face-centered magnetic components to the cell-edge + // MHDSystem::ReconstructTo(dir2edge, fc_fabs_Bx[w_comp], ec_fabs_Bi_ieside[icomp][0], ec_fabs_Bi_ieside[icomp][1], box_fc, 1, nghost_fc); + // } - // indexing: field[4: quadrant around edge] - std::array ec_fabs_E_q; - // compute the EMF along the cell-edge - for (int iquad = 0; iquad < 4; ++iquad) { - // define EMF FArrayBox - ec_fabs_E_q[iquad].resize(box_ec, 1); - // extract relevant velocity and magnetic field components - const auto &U0_qi = ec_fabs_Ui_q[0][iquad].const_array(); - const auto &U1_qi = ec_fabs_Ui_q[1][iquad].const_array(); - const auto &B0_qi = ec_fabs_Bi_ieside[0][(iquad == 0 || iquad == 3) ? 0 : 1].const_array(); - const auto &B1_qi = ec_fabs_Bi_ieside[1][(iquad < 2) ? 0 : 1].const_array(); - // compute electric field in the quadrant about the cell-edge: cross product between velocity and magnetic field in that quadrant - const auto &E2_qi = ec_fabs_E_q[iquad].array(); - amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - E2_qi(i,j,k) = U0_qi(i,j,k) * B1_qi(i,j,k) - U1_qi(i,j,k) * B0_qi(i,j,k); - }); - } + // // indexing: field[4: quadrant around edge] + // std::array ec_fabs_E_q; + // // compute the EMF along the cell-edge + // for (int iquad = 0; iquad < 4; ++iquad) { + // // define EMF FArrayBox + // ec_fabs_E_q[iquad].resize(box_ec, 1); + // // extract relevant velocity and magnetic field components + // const auto &U0_qi = ec_fabs_Ui_q[0][iquad].const_array(); + // const auto &U1_qi = ec_fabs_Ui_q[1][iquad].const_array(); + // const auto &B0_qi = ec_fabs_Bi_ieside[0][(iquad == 0 || iquad == 3) ? 0 : 1].const_array(); + // const auto &B1_qi = ec_fabs_Bi_ieside[1][(iquad < 2) ? 0 : 1].const_array(); + // // compute electric field in the quadrant about the cell-edge: cross product between velocity and magnetic field in that quadrant + // const auto &E2_qi = ec_fabs_E_q[iquad].array(); + // amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + // E2_qi(i,j,k) = U0_qi(i,j,k) * B1_qi(i,j,k) - U1_qi(i,j,k) * B0_qi(i,j,k); + // }); + // } // // extract both components of magnetic field either side of the cell-edge // const auto &B0_m = ec_fabs_Bi_ieside[0][0].const_array(); @@ -248,7 +261,7 @@ void MHDSystem::ComputeEMF(std::array::ComputeEMF(std::array -void MHDSystem::ReconstructTo(FluxDir dir, amrex::FArrayBox const &q_fab, amrex::FArrayBox &leftState_fab, amrex::FArrayBox &rightState_fab, const amrex::Box &indexRange, const int reconstructionOrder, const int nghost) +void MHDSystem::ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &indexRange, const int reconstructionOrder, const int nghost) { // N.B.: A one-zone layer around the cells must be fully reconstructed in order for PPM to work. amrex::Box const &reconstructRange = amrex::grow(indexRange, 1); amrex::Box const &x1ReconstructRange = amrex::surroundingNodes(reconstructRange, static_cast(dir)); - if (reconstructionOrder == 3) { + // if (reconstructionOrder == 3) { + // switch (dir) { + // case FluxDir::X1: + // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, x1ReconstructRange, 1); + // break; + // case FluxDir::X2: + // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, x1ReconstructRange, 1); + // break; + // case FluxDir::X3: + // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, x1ReconstructRange, 1); + // break; + // } + // } else if (reconstructionOrder == 2) { + // switch (dir) { + // case FluxDir::X1: + // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, x1ReconstructRange, 1); + // break; + // case FluxDir::X2: + // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, x1ReconstructRange, 1); + // break; + // case FluxDir::X3: + // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, x1ReconstructRange, 1); + // break; + // } + // } else if (reconstructionOrder == 1) { switch (dir) { case FluxDir::X1: - MHDSystem::template ReconstructStatesPPM(q_fab, leftState_fab, rightState_fab, reconstructRange, 1); + MHDSystem::template ReconstructStatesConstant(cState, lState, rState, x1ReconstructRange, 1); break; case FluxDir::X2: - MHDSystem::template ReconstructStatesPPM(q_fab, leftState_fab, rightState_fab, reconstructRange, 1); + MHDSystem::template ReconstructStatesConstant(cState, lState, rState, x1ReconstructRange, 1); break; case FluxDir::X3: - MHDSystem::template ReconstructStatesPPM(q_fab, leftState_fab, rightState_fab, reconstructRange, 1); + MHDSystem::template ReconstructStatesConstant(cState, lState, rState, x1ReconstructRange, 1); break; } - } else if (reconstructionOrder == 2) { - switch (dir) { - case FluxDir::X1: - MHDSystem::template ReconstructStatesPLM(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); - break; - case FluxDir::X2: - MHDSystem::template ReconstructStatesPLM(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); - break; - case FluxDir::X3: - MHDSystem::template ReconstructStatesPLM(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); - break; - } - } else if (reconstructionOrder == 1) { - switch (dir) { - case FluxDir::X1: - MHDSystem::template ReconstructStatesConstant(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); - break; - case FluxDir::X2: - MHDSystem::template ReconstructStatesConstant(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); - break; - case FluxDir::X3: - MHDSystem::template ReconstructStatesConstant(q_fab, leftState_fab, rightState_fab, x1ReconstructRange, 1); - break; - } - } else { - amrex::Abort("Invalid reconstruction order specified!"); - } + // } else { + // amrex::Abort("Invalid reconstruction order specified!"); + // } } From db4f3d1d45c0866629800e34e30298acafa70176 Mon Sep 17 00:00:00 2001 From: Neco Kriel Date: Wed, 8 May 2024 16:59:42 +1000 Subject: [PATCH 06/37] debug: add evolve step to check induction eqn --- src/FCQuantities/test_fc_quantities.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/FCQuantities/test_fc_quantities.cpp b/src/FCQuantities/test_fc_quantities.cpp index 8c21661c2..2079ac9cd 100644 --- a/src/FCQuantities/test_fc_quantities.cpp +++ b/src/FCQuantities/test_fc_quantities.cpp @@ -157,6 +157,7 @@ auto problem_main() -> int RadhydroSimulation sim_write(BCs_cc, BCs_fc); sim_write.setInitialConditions(); + sim_write.evolve(); amrex::Vector> const &state_new_fc_write = sim_write.getNewMF_fc(); amrex::Print() << "\n"; From b3fdd3a05ac3728ad69519708676900122b67a51 Mon Sep 17 00:00:00 2001 From: Neco Kriel Date: Wed, 8 May 2024 17:24:16 +1000 Subject: [PATCH 07/37] fc-rhs is only used for mhd --- src/RadhydroSimulation.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 72cf90c76..7ed1fc842 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1068,7 +1068,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); state_inter_fc_[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, nghost_fc_); - state_inter_fc_[idim].setVal(0); // prevent assert in fillBoundaryConditions when radiation is enabled + state_inter_fc_[idim].setVal(0); } } @@ -1130,12 +1130,12 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o redoFlag.setVal(quokka::redoFlag::none); if constexpr (Physics_Traits::is_mhd_enabled) { - std::array rhs_fc; + std::array rhs_mhd_fc; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); - rhs_fc[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, 0); + rhs_mhd_fc[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, 0); } - MHDSystem::ComputeEMF(rhs_fc, stateOld, stateOld_fc, dx, nghost_cc_, nghost_fc_); + MHDSystem::ComputeEMF(rhs_mhd_fc, stateOld, stateOld_fc, dx, nghost_cc_, nghost_fc_); } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); From 4ba7bc71057dbb32b11b4ff29301863987cc8e01 Mon Sep 17 00:00:00 2001 From: Neco Kriel Date: Wed, 8 May 2024 17:24:47 +1000 Subject: [PATCH 08/37] don't index comp (=1) --- src/mhd_system.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index 0dd901447..27519532f 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -51,7 +51,7 @@ void MHDSystem::ComputeEMF(std::array::bfield_index) = 0; + fcxw_a4_rhs(i, j, k) = 0; }); } @@ -277,7 +277,7 @@ void MHDSystem::ComputeEMF(std::array::bfield_index) += (E2_ave(i,j,k) - E2_ave(i-idx[0],j-idx[1],k-idx[2])) / dx[wsolve]; + // fcxw_a4_rhs(i,j,k) += (E2_ave(i,j,k) - E2_ave(i-idx[0],j-idx[1],k-idx[2])) / dx[wsolve]; // }); } } From 33a7b0ec09113e25f3c611b10666b10430649d94 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 9 May 2024 14:33:58 +1000 Subject: [PATCH 09/37] fix index-range of reconstruction (can compile and run w.o. errors -- woohoo) --- src/mhd_system.hpp | 320 +++++++++++++++++++++++---------------------- 1 file changed, 163 insertions(+), 157 deletions(-) diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index 27519532f..84cb3dce8 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -13,6 +13,7 @@ // library headers // internal headers +#include "AMReX_Print.H" #include "physics_info.hpp" #include "hydro_system.hpp" #include "hyperbolic_system.hpp" @@ -29,13 +30,13 @@ template class MHDSystem : public HyperbolicSystem::mhdFirstIndex, }; - static void ComputeEMF(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, amrex::GpuArray dx, int nghost_cc, int nghost_fc); + static void SolveInductionEqn(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars_old, std::array &fcx_mf_cVars_new, amrex::GpuArray dx, int nghost_fc, double dt); - static void ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &indexRange, int reconstructionOrder, int nghost); + static void ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &reconstructRange, int reconstructionOrder); }; template -void MHDSystem::ComputeEMF(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, amrex::GpuArray dx, const int nghost_cc, const int nghost_fc) +void MHDSystem::SolveInductionEqn(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars_old, std::array &fcx_mf_cVars_new, amrex::GpuArray dx, const int nghost_fc, const double dt) { std::cout << "Computing EMF" << std::endl; @@ -49,8 +50,10 @@ void MHDSystem::ComputeEMF(std::array::ComputeEMF(std::array cc_fabs_Ux; - cc_fabs_Ux[0].resize(amrex::grow(box_cc, nghost_cc), 1); - cc_fabs_Ux[1].resize(amrex::grow(box_cc, nghost_cc), 1); - cc_fabs_Ux[2].resize(amrex::grow(box_cc, nghost_cc), 1); + const amrex::Box &box_ccpg = amrex::grow(box_cc, nghost_fc); + cc_fabs_Ux[0].resize(box_ccpg, 1); + cc_fabs_Ux[1].resize(box_ccpg, 1); + cc_fabs_Ux[2].resize(box_ccpg, 1); const auto &cc_a4_Ux0 = cc_fabs_Ux[0].array(); const auto &cc_a4_Ux1 = cc_fabs_Ux[1].array(); const auto &cc_a4_Ux2 = cc_fabs_Ux[2].array(); const auto &cc_a4_cVars = cc_mf_cVars[mfi].const_array(); - amrex::ParallelFor(box_cc, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + amrex::ParallelFor(box_ccpg, [=] AMREX_GPU_DEVICE(int i, int j, int k) { const auto rho = cc_a4_cVars(i, j, k, HydroSystem::density_index); const auto px1 = cc_a4_cVars(i, j, k, HydroSystem::x1Momentum_index); const auto px2 = cc_a4_cVars(i, j, k, HydroSystem::x2Momentum_index); @@ -76,26 +80,26 @@ void MHDSystem::ComputeEMF(std::array fc_fabs_Bx = { - // FArrayBox(fcx_mf_cVars[0][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), - // FArrayBox(fcx_mf_cVars[1][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), - // FArrayBox(fcx_mf_cVars[2][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + // // create a view of all the b-field data (+ghost cells; do not make another copy) + // std::array fc_fabs_Bx_old = { + // FArrayBox(fcx_mf_cVars_old[0][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + // FArrayBox(fcx_mf_cVars_old[1][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + // FArrayBox(fcx_mf_cVars_old[2][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), // }; - // alternatively, make a complete copy of the b-field data - std::array fc_fabs_Bx; - for (int w_index = 0; w_index < 3; ++w_index) { - const amrex::Box box_fc = amrex::convert(box_cc, amrex::IntVect::TheDimensionVector(w_index)); - fc_fabs_Bx[w_index].resize(box_fc, 1); - const auto &fc_a4_Bx = fc_fabs_Bx[w_index].array(); + // alternatively, make a complete copy of all the b-field data (+ghost cells) + std::array fc_fabs_Bx_old; + for (int windex = 0; windex < 3; ++windex) { + const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(windex); + const amrex::Box box_fcpg = amrex::convert(amrex::grow(box_cc, nghost_fc), ivec_cc2fc); + fc_fabs_Bx_old[windex].resize(box_fcpg, 1); + const auto &fc_a4_Bx = fc_fabs_Bx_old[windex].array(); // extract face-centered magnetic fields - const auto &fc_a4_cVars = fcx_mf_cVars[w_index][mfi].const_array(); - amrex::ParallelFor(box_fc, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - fc_a4_Bx(i, j, k) = fc_a4_cVars(i, j, k, MHDSystem::bfield_index); + const auto &fc_a4_cVars_old = fcx_mf_cVars_old[windex][mfi].const_array(); + amrex::ParallelFor(box_fcpg, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + fc_a4_Bx(i, j, k) = fc_a4_cVars_old(i, j, k, MHDSystem::bfield_index); }); } - // compute the magnetic flux through each cell-face for (int wsolve = 0; wsolve < 3; ++wsolve) { // electric field on the cell-edge @@ -112,10 +116,15 @@ void MHDSystem::ComputeEMF(std::arrayfc->ec + const amrex::Box box_ecpgm2 = amrex::grow(box_ec, nghost_fc-2); + // you lose 1 ghost-cell when you reconstruct b-field fc->ec + const amrex::Box box_ecpgm1 = amrex::grow(box_ec, nghost_fc-1); + // define output electric field on the cell-edge - eci_fabs_E[iedge_rel2face].resize(box_ec, 1); + eci_fabs_E[iedge_rel2face].resize(box_ecpgm2, 1); // initialise FArrayBox for storing the edge-centered velocity fields averaged across the two extrapolation permutations // indexing: field[2: i-compnent][4: quadrant around edge] @@ -125,17 +134,17 @@ void MHDSystem::ComputeEMF(std::array ec_fabs_U_ieside; // define the four possible velocity field quantities that could be reconstructed at the cell-edge // also define the temporary velocity field quantities that will be used for computing the extrapolation - ec_fabs_U_ieside[0].resize(box_ec, 1); - ec_fabs_U_ieside[1].resize(box_ec, 1); + ec_fabs_U_ieside[0].resize(box_ecpgm2, 1); + ec_fabs_U_ieside[1].resize(box_ecpgm2, 1); // indexing: field[2: i-compnent][2: i-side of edge] // note: magnetic field components cannot be discontinuous along themselves (i.e., either side of the face where they are stored) std::array, 2> ec_fabs_Bi_ieside; // define quantities for (int icomp = 0; icomp < 2; ++icomp) { - ec_fabs_Bi_ieside[icomp][0].resize(box_ec, 1); - ec_fabs_Bi_ieside[icomp][1].resize(box_ec, 1); + ec_fabs_Bi_ieside[icomp][0].resize(box_ecpgm1, 1); + ec_fabs_Bi_ieside[icomp][1].resize(box_ecpgm1, 1); for (int iquad = 0; iquad < 4; ++iquad) { - ec_fabs_Ui_q[icomp][iquad].resize(box_ec, 1); + ec_fabs_Ui_q[icomp][iquad].resize(box_ecpgm2, 1); } } @@ -148,7 +157,8 @@ void MHDSystem::ComputeEMF(std::array(w_extrap_dir2face); // define extrapolation direction to go from face to edge @@ -161,125 +171,120 @@ void MHDSystem::ComputeEMF(std::array::template ReconstructStatesConstant(cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), fc_fabs_Ui_ifside[icomp][1].array(), box_fc, 1); - // break; - // case FluxDir::X2: - // std::cout << "x2" << std::endl; - // MHDSystem::template ReconstructStatesConstant(cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), fc_fabs_Ui_ifside[icomp][1].array(), box_fc, 1); - // break; - // case FluxDir::X3: - // std::cout << "x3" << std::endl; - // MHDSystem::template ReconstructStatesConstant(cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), fc_fabs_Ui_ifside[icomp][1].array(), box_fc, 1); - // break; - // } - // MHDSystem::ReconstructTo(dir2face, cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), fc_fabs_Ui_ifside[icomp][1].array(), box_cc, 1, nghost_cc); - // // extrapolate face-centered velocity components to the cell-edge - // for (int iface = 0; iface < 2; ++iface) { - // // reset values in temporary FArrayBox - // ec_fabs_U_ieside[0].setVal(0.0); - // ec_fabs_U_ieside[1].setVal(0.0); - // // extrapolate face-centered velocity component to the cell-edge - // MHDSystem::ReconstructTo(dir2edge, fc_fabs_Ui_ifside[icomp][iface], ec_fabs_U_ieside[0], ec_fabs_U_ieside[1], box_fc, 1, nghost_cc); - // // figure out which quadrant of the cell-edge this extrapolated velocity component corresponds with - // int iquad0 = -1; - // int iquad1 = -1; - // // note: quadrants are defined based on where the quantity sits relative to the edge (dir-0, dir-1): - // // (-,+) | (+,+) - // // 1 | 2 - // // ------+------ - // // 0 | 3 - // // (-,-) | (+,-) - // if (iperm == 0) { - // iquad0 = (iface == 0) ? 0 : 3; - // iquad1 = (iface == 0) ? 1 : 2; - // } else { - // iquad0 = (iface == 0) ? 0 : 1; - // iquad1 = (iface == 0) ? 3 : 2; - // } - // ec_fabs_Ui_q[icomp][iquad0].atomicAdd(ec_fabs_U_ieside[0], 0, 0, 1); - // ec_fabs_Ui_q[icomp][iquad1].atomicAdd(ec_fabs_U_ieside[1], 0, 0, 1); - // } + fc_fabs_Ui_ifside[icomp][0].resize(box_fcpgm1, 1); + fc_fabs_Ui_ifside[icomp][1].resize(box_fcpgm1, 1); + // extrapolate cell-centered velocity components to the cell-face + MHDSystem::ReconstructTo(dir2face, cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), fc_fabs_Ui_ifside[icomp][1].array(), box_fcpgm1, 1); + // extrapolate face-centered velocity components to the cell-edge + for (int iface = 0; iface < 2; ++iface) { + // reset values in temporary FArrayBox + ec_fabs_U_ieside[0].setVal(0.0); + ec_fabs_U_ieside[1].setVal(0.0); + // extrapolate face-centered velocity component to the cell-edge + MHDSystem::ReconstructTo(dir2edge, fc_fabs_Ui_ifside[icomp][iface].array(), ec_fabs_U_ieside[0].array(), ec_fabs_U_ieside[1].array(), box_ecpgm2, 1); + // figure out which quadrant of the cell-edge this extrapolated velocity component corresponds with + int iquad0 = -1; + int iquad1 = -1; + // note: quadrants are defined based on where the quantity sits relative to the edge (dir-0, dir-1): + // (-,+) | (+,+) + // 1 | 2 + // ------+------ + // 0 | 3 + // (-,-) | (+,-) + if (iperm == 0) { + iquad0 = (iface == 0) ? 0 : 3; + iquad1 = (iface == 0) ? 1 : 2; + } else { + iquad0 = (iface == 0) ? 0 : 1; + iquad1 = (iface == 0) ? 3 : 2; + } + ec_fabs_Ui_q[icomp][iquad0].atomicAdd(ec_fabs_U_ieside[0], 0, 0, 1); + ec_fabs_Ui_q[icomp][iquad1].atomicAdd(ec_fabs_U_ieside[1], 0, 0, 1); + } + } + } + // finish averaging the two different ways for extrapolating cc->ec velocity fields + for (int icomp = 0; icomp < 2; ++icomp) { + for (int iquad = 0; iquad < 4; ++iquad) { + ec_fabs_Ui_q[icomp][iquad].mult(0.5, 0, 1); } } - // // finish averaging the two different ways for extrapolating cc->ec velocity fields - // for (int icomp = 0; icomp < 2; ++icomp) { - // for (int iquad = 0; iquad < 4; ++iquad) { - // ec_fabs_Ui_q[icomp][iquad].mult(0.5, 0, 1); - // } - // } - // // extrapolate the two required face-centered magnetic field components to the cell-edge - // for (int icomp = 0; icomp < 2; ++icomp) { - // // define extrapolation direction to go from face to edge - // const int w_comp = w_extrap_dirs[icomp]; - // const int w_extrap_dir2edge = w_extrap_dirs[(icomp+1) % 2]; - // const auto dir2edge = static_cast(w_extrap_dir2edge); - // const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(w_extrap_dir2edge); - // const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); - // // extrapolate face-centered magnetic components to the cell-edge - // MHDSystem::ReconstructTo(dir2edge, fc_fabs_Bx[w_comp], ec_fabs_Bi_ieside[icomp][0], ec_fabs_Bi_ieside[icomp][1], box_fc, 1, nghost_fc); - // } + // extrapolate the two required face-centered magnetic field components to the cell-edge + for (int icomp = 0; icomp < 2; ++icomp) { + // define extrapolation direction to go from face to edge + const int w_comp = w_extrap_dirs[icomp]; + const int w_extrap_dir2edge = w_extrap_dirs[(icomp+1) % 2]; + const auto dir2edge = static_cast(w_extrap_dir2edge); + const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(w_extrap_dir2edge); + const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); + // extrapolate face-centered magnetic components to the cell-edge + MHDSystem::ReconstructTo(dir2edge, fc_fabs_Bx_old[w_comp].array(), ec_fabs_Bi_ieside[icomp][0].array(), ec_fabs_Bi_ieside[icomp][1].array(), box_ecpgm1, 1); + } - // // indexing: field[4: quadrant around edge] - // std::array ec_fabs_E_q; - // // compute the EMF along the cell-edge - // for (int iquad = 0; iquad < 4; ++iquad) { - // // define EMF FArrayBox - // ec_fabs_E_q[iquad].resize(box_ec, 1); - // // extract relevant velocity and magnetic field components - // const auto &U0_qi = ec_fabs_Ui_q[0][iquad].const_array(); - // const auto &U1_qi = ec_fabs_Ui_q[1][iquad].const_array(); - // const auto &B0_qi = ec_fabs_Bi_ieside[0][(iquad == 0 || iquad == 3) ? 0 : 1].const_array(); - // const auto &B1_qi = ec_fabs_Bi_ieside[1][(iquad < 2) ? 0 : 1].const_array(); - // // compute electric field in the quadrant about the cell-edge: cross product between velocity and magnetic field in that quadrant - // const auto &E2_qi = ec_fabs_E_q[iquad].array(); - // amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - // E2_qi(i,j,k) = U0_qi(i,j,k) * B1_qi(i,j,k) - U1_qi(i,j,k) * B0_qi(i,j,k); - // }); - // } + // indexing: field[4: quadrant around edge] + std::array ec_fabs_E_q; + // compute the EMF along the cell-edge + for (int iquad = 0; iquad < 4; ++iquad) { + // define EMF FArrayBox + ec_fabs_E_q[iquad].resize(box_ecpgm2, 1); + // extract relevant velocity and magnetic field components + const auto &U0_qi = ec_fabs_Ui_q[0][iquad].const_array(); + const auto &U1_qi = ec_fabs_Ui_q[1][iquad].const_array(); + const auto &B0_qi = ec_fabs_Bi_ieside[0][(iquad == 0 || iquad == 3) ? 0 : 1].const_array(); + const auto &B1_qi = ec_fabs_Bi_ieside[1][(iquad < 2) ? 0 : 1].const_array(); + // compute electric field in the quadrant about the cell-edge: cross product between velocity and magnetic field in that quadrant + const auto &E2_qi = ec_fabs_E_q[iquad].array(); + amrex::ParallelFor(box_ecpgm2, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + E2_qi(i,j,k) = U0_qi(i,j,k) * B1_qi(i,j,k) - U1_qi(i,j,k) * B0_qi(i,j,k); + }); + } - // // extract both components of magnetic field either side of the cell-edge - // const auto &B0_m = ec_fabs_Bi_ieside[0][0].const_array(); - // const auto &B0_p = ec_fabs_Bi_ieside[0][1].const_array(); - // const auto &B1_m = ec_fabs_Bi_ieside[1][0].const_array(); - // const auto &B1_p = ec_fabs_Bi_ieside[1][1].const_array(); - // // extract all four quadrants of the electric field about the cell-edge - // const auto &E2_q0 = ec_fabs_E_q[0].const_array(); - // const auto &E2_q1 = ec_fabs_E_q[1].const_array(); - // const auto &E2_q2 = ec_fabs_E_q[2].const_array(); - // const auto &E2_q3 = ec_fabs_E_q[3].const_array(); - // // compute electric field on the cell-edge - // const auto &E2_ave = eci_fabs_E[iedge_rel2face].array(); - // amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - // const double spd_x0_m = 1.0; - // const double spd_x0_p = 1.0; - // const double spd_x1_m = 1.0; - // const double spd_x1_p = 1.0; - // E2_ave(i,j,k) = - // spd_x0_p * spd_x1_p * E2_q0(i,j,k) + - // spd_x0_m * spd_x1_p * E2_q3(i,j,k) + - // spd_x0_m * spd_x1_m * E2_q1(i,j,k) + - // spd_x0_p * spd_x1_m * E2_q2(i,j,k) - - // spd_x1_p * spd_x1_m / (spd_x1_p + spd_x1_m) * (B0_p(i,j,k) - B0_m(i,j,k)) + - // spd_x0_p * spd_x0_m / (spd_x0_p + spd_x0_m) * (B1_p(i,j,k) - B1_m(i,j,k)); - // }); + // extract both components of magnetic field either side of the cell-edge + const auto &B0_m = ec_fabs_Bi_ieside[0][0].const_array(); + const auto &B0_p = ec_fabs_Bi_ieside[0][1].const_array(); + const auto &B1_m = ec_fabs_Bi_ieside[1][0].const_array(); + const auto &B1_p = ec_fabs_Bi_ieside[1][1].const_array(); + // extract all four quadrants of the electric field about the cell-edge + const auto &E2_q0 = ec_fabs_E_q[0].const_array(); + const auto &E2_q1 = ec_fabs_E_q[1].const_array(); + const auto &E2_q2 = ec_fabs_E_q[2].const_array(); + const auto &E2_q3 = ec_fabs_E_q[3].const_array(); + // compute electric field on the cell-edge + const auto &E2_ave = eci_fabs_E[iedge_rel2face].array(); + amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + const double spd_x0_m = 1.0; + const double spd_x0_p = 1.0; + const double spd_x1_m = 1.0; + const double spd_x1_p = 1.0; + E2_ave(i,j,k) = + spd_x0_p * spd_x1_p * E2_q0(i,j,k) + + spd_x0_m * spd_x1_p * E2_q3(i,j,k) + + spd_x0_m * spd_x1_m * E2_q1(i,j,k) + + spd_x0_p * spd_x1_m * E2_q2(i,j,k) - + spd_x1_p * spd_x1_m / (spd_x1_p + spd_x1_m) * (B0_p(i,j,k) - B0_m(i,j,k)) + + spd_x0_p * spd_x0_m / (spd_x0_p + spd_x0_m) * (B1_p(i,j,k) - B1_m(i,j,k)); + }); - // const int index_E2comp = MHDSystem::bfield_index + (wsolve + 2*iedge_rel2face + 2) % 3; - // std::array idx = {0, 0, 0}; - // idx[index_E2comp] = 1; - // // compute the magnetic flux - // const auto &fcxw_a4_rhs = fcx_mf_rhs[wsolve][mfi].array(); - // amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - // // the induction equation is written in an additive form: the RHS is evaluated in parts as each edge-centered electric field is computed - // fcxw_a4_rhs(i,j,k) += (E2_ave(i,j,k) - E2_ave(i-idx[0],j-idx[1],k-idx[2])) / dx[wsolve]; - // }); + const int index_E2comp = MHDSystem::bfield_index + (wsolve + 2*iedge_rel2face + 2) % 3; + std::array idx = {0, 0, 0}; + idx[index_E2comp] = 1; + // compute the magnetic flux + const auto &fcxw_a4_rhs = fcx_mf_rhs[wsolve][mfi].array(); + amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + // the induction equation is written in an additive form: the RHS is evaluated in parts as each edge-centered electric field is computed + fcxw_a4_rhs(i,j,k) += (E2_ave(i,j,k) - E2_ave(i-idx[0],j-idx[1],k-idx[2])) / dx[wsolve]; + }); } + + const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(wsolve); + const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); + const auto &fcxw_a4_rhs = fcx_mf_rhs[wsolve][mfi].const_array(); + const auto &fc_a4_Bx_old = fc_fabs_Bx_old[wsolve].const_array(); + auto fc_a4_Bx_new = fcx_mf_cVars_new[wsolve][mfi].array(); + amrex::ParallelFor(box_fc, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + fc_a4_Bx_new(i, j, k) = fc_a4_Bx_old(i, j, k) + dt * fcxw_a4_rhs(i, j, k); + }); } } @@ -287,46 +292,47 @@ void MHDSystem::ComputeEMF(std::array -void MHDSystem::ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &indexRange, const int reconstructionOrder, const int nghost) +void MHDSystem::ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &reconstructRange, const int reconstructionOrder) { - // N.B.: A one-zone layer around the cells must be fully reconstructed in order for PPM to work. - amrex::Box const &reconstructRange = amrex::grow(indexRange, 1); - amrex::Box const &x1ReconstructRange = amrex::surroundingNodes(reconstructRange, static_cast(dir)); + // // N.B.: A one-zone layer around the cells must be fully reconstructed in order for PPM to work. + // amrex::Box const &reconstructRange = amrex::convert(indexRange, amrex::IntVect::TheDimensionVector(static_cast(dir))); // if (reconstructionOrder == 3) { // switch (dir) { // case FluxDir::X1: - // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, x1ReconstructRange, 1); + // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); // break; // case FluxDir::X2: - // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, x1ReconstructRange, 1); + // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); // break; // case FluxDir::X3: - // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, x1ReconstructRange, 1); + // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); // break; // } // } else if (reconstructionOrder == 2) { // switch (dir) { // case FluxDir::X1: - // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, x1ReconstructRange, 1); + // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); // break; // case FluxDir::X2: - // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, x1ReconstructRange, 1); + // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); // break; // case FluxDir::X3: - // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, x1ReconstructRange, 1); + // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); // break; // } // } else if (reconstructionOrder == 1) { + // amrex::Print() << "here." << std::endl; + // amrex::Print() << "Size of cState: " << cState.size() << std::endl; switch (dir) { case FluxDir::X1: - MHDSystem::template ReconstructStatesConstant(cState, lState, rState, x1ReconstructRange, 1); + MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); break; case FluxDir::X2: - MHDSystem::template ReconstructStatesConstant(cState, lState, rState, x1ReconstructRange, 1); + MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); break; case FluxDir::X3: - MHDSystem::template ReconstructStatesConstant(cState, lState, rState, x1ReconstructRange, 1); + MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); break; } // } else { From 4309caaf06ef32a56778c4df46472332bc591259 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 9 May 2024 14:34:37 +1000 Subject: [PATCH 10/37] fix arg parsing to induction soln --- src/RadhydroSimulation.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 7ed1fc842..4f84b5b5a 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1116,7 +1116,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto &stateNew = state_inter_cc_; auto const &stateOld_fc = state_old_fc_tmp; - // auto &stateNew_fc = state_inter_fc_; + auto &stateNew_fc = state_inter_fc_; auto [fluxArrays, faceVel] = computeHydroFluxes(stateOld, ncompHydro_, lev); @@ -1130,12 +1130,12 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o redoFlag.setVal(quokka::redoFlag::none); if constexpr (Physics_Traits::is_mhd_enabled) { - std::array rhs_mhd_fc; + std::array rhs4mhd_fc; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); - rhs_mhd_fc[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, 0); + rhs4mhd_fc[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, nghost_fc_); } - MHDSystem::ComputeEMF(rhs_mhd_fc, stateOld, stateOld_fc, dx, nghost_cc_, nghost_fc_); + MHDSystem::SolveInductionEqn(rhs4mhd_fc, stateOld, stateOld_fc, stateNew_fc, dx, nghost_fc_, dt_lev); } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); From d2c3bd4cacc0a0710cca1dba2881ea1bb32f4692 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 9 May 2024 14:35:05 +1000 Subject: [PATCH 11/37] recuce domain size for terminal debugging --- tests/fc_hydro_wave.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fc_hydro_wave.in b/tests/fc_hydro_wave.in index 9d3a5923c..da4d0415c 100644 --- a/tests/fc_hydro_wave.in +++ b/tests/fc_hydro_wave.in @@ -13,7 +13,7 @@ amr.v = 0 # verbosity in Amr # ***************************************************************** # Resolution and refinement # ***************************************************************** -amr.n_cell = 100 40 4 +amr.n_cell = 12 8 4 amr.max_level = 0 # number of levels = max_level + 1 amr.blocking_factor = 4 # grid size must be divisible by this From 65f747af2dd334c5c34a02834cc2aa13477c206b Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 9 May 2024 14:36:36 +1000 Subject: [PATCH 12/37] specify state centering --- src/RadhydroSimulation.hpp | 94 +++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 4f84b5b5a..21eb6e93d 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -238,7 +238,7 @@ template class RadhydroSimulation : public AMRSimulation bool; // radiation subcycle - void swapRadiationState(amrex::MultiFab &stateOld, amrex::MultiFab const &stateNew); + void swapRadiationState(amrex::MultiFab &stateOld_cc, amrex::MultiFab const &stateNew_cc); auto computeNumberOfRadiationSubsteps(int lev, amrex::Real dt_lev_hydro) -> int; void advanceRadiationSubstepAtLevel(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine); @@ -250,7 +250,7 @@ template class RadhydroSimulation : public AMRSimulation const &stateNew, const amrex::Box &indexRange, amrex::Real time, double dt, int stage, + void operatorSplitSourceTerms(amrex::Array4 const &stateNew_cc, const amrex::Box &indexRange, amrex::Real time, double dt, int stage, amrex::GpuArray const &dx, amrex::GpuArray const &prob_lo, amrex::GpuArray const &prob_hi); @@ -410,19 +410,19 @@ template void RadhydroSimulation::computeMaxSign // hydro: loop over local grids, compute CFL timestep for (amrex::MFIter iter(state_new_cc_[level]); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); - auto const &stateNew = state_new_cc_[level].const_array(iter); + auto const &stateNew_cc = state_new_cc_[level].const_array(iter); auto const &maxSignal = max_signal_speed_[level].array(iter); if constexpr (Physics_Traits::is_hydro_enabled && !(Physics_Traits::is_radiation_enabled)) { // hydro only - HydroSystem::ComputeMaxSignalSpeed(stateNew, maxSignal, indexRange); + HydroSystem::ComputeMaxSignalSpeed(stateNew_cc, maxSignal, indexRange); } else if constexpr (Physics_Traits::is_radiation_enabled) { // radiation hydro, or radiation only - RadSystem::ComputeMaxSignalSpeed(stateNew, maxSignal, indexRange); + RadSystem::ComputeMaxSignalSpeed(stateNew_cc, maxSignal, indexRange); if constexpr (Physics_Traits::is_hydro_enabled) { auto maxSignalHydroFAB = amrex::FArrayBox(indexRange); auto const &maxSignalHydro = maxSignalHydroFAB.array(); - HydroSystem::ComputeMaxSignalSpeed(stateNew, maxSignalHydro, indexRange); + HydroSystem::ComputeMaxSignalSpeed(stateNew_cc, maxSignalHydro, indexRange); const int maxSubsteps = maxSubsteps_; // ensure that we use the smaller of the two timesteps amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { @@ -1112,13 +1112,13 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // Stage 1 of RK2-SSP { // advance all grids on local processor (Stage 1 of integrator) - auto const &stateOld = state_old_cc_tmp; - auto &stateNew = state_inter_cc_; + auto const &stateOld_cc = state_old_cc_tmp; + auto &stateNew_cc = state_inter_cc_; auto const &stateOld_fc = state_old_fc_tmp; auto &stateNew_fc = state_inter_fc_; - auto [fluxArrays, faceVel] = computeHydroFluxes(stateOld, ncompHydro_, lev); + auto [fluxArrays, faceVel] = computeHydroFluxes(stateOld_cc, ncompHydro_, lev); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); @@ -1135,11 +1135,11 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); rhs4mhd_fc[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, nghost_fc_); } - MHDSystem::SolveInductionEqn(rhs4mhd_fc, stateOld, stateOld_fc, stateNew_fc, dx, nghost_fc_, dt_lev); + MHDSystem::SolveInductionEqn(rhs4mhd_fc, stateOld_cc, stateOld_fc, stateNew_fc, dx, nghost_fc_, dt_lev); } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, faceVel, redoFlag); + HydroSystem::PredictStep(stateOld_cc, stateNew_cc, rhs, dt_lev, ncompHydro_, redoFlag); // LOW LEVEL DEBUGGING: output rhs if (lowLevelDebuggingOutput_ == 1) { @@ -1176,7 +1176,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o const amrex::IntVect cell_idx = redoFlag.maxIndex(0); // Calculate the coordinates based on the cell index and cell size printCoordinates(lev, cell_idx); - amrex::print_state(stateNew, cell_idx); + amrex::print_state(stateNew_cc, cell_idx); } // synchronize redoFlag across ranks @@ -1188,8 +1188,8 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // re-do RK update HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, faceVel, redoFlag); + HydroSystem::PredictStep(stateOld_cc, stateNew_cc, rhs, dt_lev, ncompHydro_, redoFlag); amrex::Gpu::streamSynchronizeAll(); // just in case amrex::Long const ncells_bad = static_cast(redoFlag.sum(0)); @@ -1200,7 +1200,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // print cell state amrex::Print() << "[FOFC-1] Flux correction failed:\n"; printCoordinates(lev, cell_idx); - amrex::print_state(stateNew, cell_idx); + amrex::print_state(stateNew_cc, cell_idx); amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; } if (abortOnFofcFailure_ != 0) { @@ -1210,11 +1210,11 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } // prevent vacuum - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew_cc); if (useDualEnergy_ == 1) { // sync internal energy (requires positive density) - HydroSystem::SyncDualEnergy(stateNew); + HydroSystem::SyncDualEnergy(stateNew_cc); } if (do_reflux == 1) { @@ -1234,7 +1234,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o AMREX_ASSERT(!state_inter_cc_.contains_nan(0, state_inter_cc_.nComp())); AMREX_ASSERT(!state_inter_cc_.contains_nan()); // check ghost zones - auto const &stateOld = state_old_cc_tmp; + auto const &stateOld_cc = state_old_cc_tmp; auto const &stateInter = state_inter_cc_; auto &stateFinal = state_new_cc_[lev]; auto [fluxArrays, faceVel] = computeHydroFluxes(stateInter, ncompHydro_, lev); @@ -1249,8 +1249,8 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o redoFlag.setVal(quokka::redoFlag::none); HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, avgFaceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, avgFaceVel, redoFlag); + HydroSystem::PredictStep(stateOld_cc, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); // do first-order flux correction (FOFC) amrex::Gpu::streamSynchronizeAll(); // just in case @@ -1272,8 +1272,8 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // re-do RK update HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, avgFaceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, avgFaceVel, redoFlag); + HydroSystem::PredictStep(stateOld_cc, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); amrex::Gpu::streamSynchronizeAll(); // just in case amrex::Long ncells_bad = redoFlag.sum(0); @@ -1590,10 +1590,10 @@ void RadhydroSimulation::hydroFOFluxFunction(amrex::MultiFab const &p HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); } -template void RadhydroSimulation::swapRadiationState(amrex::MultiFab &stateOld, amrex::MultiFab const &stateNew) +template void RadhydroSimulation::swapRadiationState(amrex::MultiFab &stateOld_cc, amrex::MultiFab const &stateNew_cc) { - // copy radiation state variables from stateNew to stateOld - amrex::MultiFab::Copy(stateOld, stateNew, nstartHyperbolic_, nstartHyperbolic_, ncompHyperbolic_, 0); + // copy radiation state variables from stateNew_cc to stateOld_cc + amrex::MultiFab::Copy(stateOld_cc, stateNew_cc, nstartHyperbolic_, nstartHyperbolic_, ncompHyperbolic_, 0); } template @@ -1644,13 +1644,13 @@ void RadhydroSimulation::subcycleRadiationAtLevel(int lev, amrex::Rea // matter-radiation exchange source terms of stage 1 for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); - auto const &stateNew = state_new_cc_[lev].array(iter); + auto const &stateNew_cc = state_new_cc_[lev].array(iter); auto const &prob_lo = geom[lev].ProbLoArray(); auto const &prob_hi = geom[lev].ProbHiArray(); // update state_new_cc_[lev] in place (updates both radiation and hydro vars) // Note that only a fraction (IMEX_a32) of the matter-radiation exchange source terms are added to hydro. This ensures that the // hydro properties get to t + IMEX_a32 dt in terms of matter-radiation exchange. - operatorSplitSourceTerms(stateNew, indexRange, time_subcycle, dt_radiation, 1, dx, prob_lo, prob_hi); + operatorSplitSourceTerms(stateNew_cc, indexRange, time_subcycle, dt_radiation, 1, dx, prob_lo, prob_hi); } } @@ -1663,11 +1663,11 @@ void RadhydroSimulation::subcycleRadiationAtLevel(int lev, amrex::Rea // Add the matter-radiation exchange source terms to the radiation subsystem and evolve by (1 - IMEX_a32) * dt for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); - auto const &stateNew = state_new_cc_[lev].array(iter); + auto const &stateNew_cc = state_new_cc_[lev].array(iter); auto const &prob_lo = geom[lev].ProbLoArray(); auto const &prob_hi = geom[lev].ProbHiArray(); // update state_new_cc_[lev] in place (updates both radiation and hydro vars) - operatorSplitSourceTerms(stateNew, indexRange, time_subcycle, dt_radiation, 2, dx, prob_lo, prob_hi); + operatorSplitSourceTerms(stateNew_cc, indexRange, time_subcycle, dt_radiation, 2, dx, prob_lo, prob_hi); } // new hydro+radiation state is stored in state_new_cc_ @@ -1702,13 +1702,13 @@ void RadhydroSimulation::advanceRadiationSubstepAtLevel(int lev, amre // advance all grids on local processor (Stage 1 of integrator) for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); + auto const &stateOld_cc = state_old_cc_[lev].const_array(iter); + auto const &stateNew_cc = state_new_cc_[lev].array(iter); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld_cc, indexRange, ncompHyperbolic_, dx); // Stage 1 of RK2-SSP RadSystem::PredictStep( - stateOld, stateNew, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + stateOld_cc, stateNew_cc, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, dt_radiation, dx, indexRange, ncompHyperbolic_); @@ -1727,14 +1727,14 @@ void RadhydroSimulation::advanceRadiationSubstepAtLevel(int lev, amre // advance all grids on local processor (Stage 2 of integrator) for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); + auto const &stateOld_cc = state_old_cc_[lev].const_array(iter); auto const &stateInter = state_new_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); + auto const &stateNew_cc = state_new_cc_[lev].array(iter); auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); // Stage 2 of RK2-SSP RadSystem::AddFluxesRK2( - stateNew, stateOld, stateInter, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + stateNew_cc, stateOld_cc, stateInter, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, dt_radiation, dx, indexRange, ncompHyperbolic_); @@ -1761,13 +1761,13 @@ void RadhydroSimulation::advanceRadiationForwardEuler(int lev, amrex: // advance all grids on local processor (Stage 1 of integrator) for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); + auto const &stateOld_cc = state_old_cc_[lev].const_array(iter); + auto const &stateNew_cc = state_new_cc_[lev].array(iter); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld_cc, indexRange, ncompHyperbolic_, dx); // Stage 1 of RK2-SSP RadSystem::PredictStep( - stateOld, stateNew, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + stateOld_cc, stateNew_cc, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, dt_radiation, dx, indexRange, ncompHyperbolic_); @@ -1793,15 +1793,15 @@ void RadhydroSimulation::advanceRadiationMidpointRK2(int lev, amrex:: // advance all grids on local processor (Stage 2 of integrator) for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); + auto const &stateOld_cc = state_old_cc_[lev].const_array(iter); auto const &stateInter = state_new_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArraysOld, fluxDiffusiveArraysOld] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); + auto const &stateNew_cc = state_new_cc_[lev].array(iter); + auto [fluxArraysOld, fluxDiffusiveArraysOld] = computeRadiationFluxes(stateOld_cc, indexRange, ncompHyperbolic_, dx); auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); // Stage 2 of RK2-SSP RadSystem::AddFluxesRK2( - stateNew, stateOld, stateInter, {AMREX_D_DECL(fluxArraysOld[0].array(), fluxArraysOld[1].array(), fluxArraysOld[2].array())}, + stateNew_cc, stateOld_cc, stateInter, {AMREX_D_DECL(fluxArraysOld[0].array(), fluxArraysOld[1].array(), fluxArraysOld[2].array())}, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, {AMREX_D_DECL(fluxDiffusiveArraysOld[0].const_array(), fluxDiffusiveArraysOld[1].const_array(), fluxDiffusiveArraysOld[2].const_array())}, {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, @@ -1817,7 +1817,7 @@ void RadhydroSimulation::advanceRadiationMidpointRK2(int lev, amrex:: } template -void RadhydroSimulation::operatorSplitSourceTerms(amrex::Array4 const &stateNew, const amrex::Box &indexRange, const amrex::Real time, +void RadhydroSimulation::operatorSplitSourceTerms(amrex::Array4 const &stateNew_cc, const amrex::Box &indexRange, const amrex::Real time, const double dt, const int stage, amrex::GpuArray const &dx, amrex::GpuArray const &prob_lo, amrex::GpuArray const &prob_hi) @@ -1831,7 +1831,7 @@ void RadhydroSimulation::operatorSplitSourceTerms(amrex::Array4::SetRadEnergySource(radEnergySource.array(), indexRange, dx, prob_lo, prob_hi, time + dt); // cell-centered source terms - RadSystem::AddSourceTerms(stateNew, radEnergySource.const_array(), indexRange, dt, stage); + RadSystem::AddSourceTerms(stateNew_cc, radEnergySource.const_array(), indexRange, dt, stage); } template From 03e6001aa2cc3fbb86c378d4a507859675bce226 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 9 May 2024 16:13:48 +1000 Subject: [PATCH 13/37] reinstate all reconstruct options --- src/mhd_system.hpp | 73 +++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index 84cb3dce8..f73b3827e 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -102,6 +102,8 @@ void MHDSystem::SolveInductionEqn(std::array eci_fabs_E; @@ -252,12 +254,12 @@ void MHDSystem::SolveInductionEqn(std::array::SolveInductionEqn(std::array::SolveInductionEqn(std::array void MHDSystem::ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &reconstructRange, const int reconstructionOrder) { - // // N.B.: A one-zone layer around the cells must be fully reconstructed in order for PPM to work. - // amrex::Box const &reconstructRange = amrex::convert(indexRange, amrex::IntVect::TheDimensionVector(static_cast(dir))); - - // if (reconstructionOrder == 3) { - // switch (dir) { - // case FluxDir::X1: - // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); - // break; - // case FluxDir::X2: - // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); - // break; - // case FluxDir::X3: - // MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); - // break; - // } - // } else if (reconstructionOrder == 2) { - // switch (dir) { - // case FluxDir::X1: - // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); - // break; - // case FluxDir::X2: - // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); - // break; - // case FluxDir::X3: - // MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); - // break; - // } - // } else if (reconstructionOrder == 1) { - // amrex::Print() << "here." << std::endl; - // amrex::Print() << "Size of cState: " << cState.size() << std::endl; + if (reconstructionOrder == 3) { + switch (dir) { + case FluxDir::X1: + MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); + break; + case FluxDir::X2: + MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); + break; + case FluxDir::X3: + MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); + break; + } + } else if (reconstructionOrder == 2) { + switch (dir) { + case FluxDir::X1: + MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); + break; + case FluxDir::X2: + MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); + break; + case FluxDir::X3: + MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); + break; + } + } else if (reconstructionOrder == 1) { switch (dir) { case FluxDir::X1: MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); @@ -335,9 +330,9 @@ void MHDSystem::ReconstructTo(FluxDir dir, arrayconst_t &cState, arra MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); break; } - // } else { - // amrex::Abort("Invalid reconstruction order specified!"); - // } + } else { + amrex::Abort("Invalid reconstruction order specified!"); + } } From ec40a303e89939b5a3d7325293f1a4de96dfe4f2 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 6 Jun 2024 15:17:02 +1000 Subject: [PATCH 14/37] add alfven wave test problem (not implemented yet) --- src/AlfvenWave/CMakeLists.txt | 7 ++ src/AlfvenWave/test_alfven_wave.cpp | 174 ++++++++++++++++++++++++++++ src/AlfvenWave/test_alfven_wave.hpp | 20 ++++ tests/alfven_wave.in | 24 ++++ 4 files changed, 225 insertions(+) create mode 100644 src/AlfvenWave/CMakeLists.txt create mode 100644 src/AlfvenWave/test_alfven_wave.cpp create mode 100644 src/AlfvenWave/test_alfven_wave.hpp create mode 100644 tests/alfven_wave.in diff --git a/src/AlfvenWave/CMakeLists.txt b/src/AlfvenWave/CMakeLists.txt new file mode 100644 index 000000000..ac6534750 --- /dev/null +++ b/src/AlfvenWave/CMakeLists.txt @@ -0,0 +1,7 @@ +add_executable(test_alfven_wave test_alfven_wave.cpp ../interpolate.cpp ${QuokkaObjSources}) + +if(AMReX_GPU_BACKEND MATCHES "CUDA") + setup_target_for_cuda_compilation(test_alfven_wave) +endif() + +add_test(NAME AlfvenWave COMMAND test_alfven_wave alfven_wave.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) diff --git a/src/AlfvenWave/test_alfven_wave.cpp b/src/AlfvenWave/test_alfven_wave.cpp new file mode 100644 index 000000000..81d21928c --- /dev/null +++ b/src/AlfvenWave/test_alfven_wave.cpp @@ -0,0 +1,174 @@ +//============================================================================== +// Copyright 2022 Neco Kriel. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file test_fc_quantities.cpp +/// \brief Defines a test problem to make sure face-centred quantities are created correctly. +/// + +#include +#include +#include +#include + +#include "AMReX_Array.H" +#include "AMReX_Array4.H" +#include "AMReX_Print.H" +#include "AMReX_REAL.H" + +#include "RadhydroSimulation.hpp" +#include "fextract.hpp" +#include "grid.hpp" +#include "physics_info.hpp" +#include "test_alfven_wave.hpp" + +struct AlfvenWave { +}; + +template <> struct quokka::EOS_Traits { + static constexpr double gamma = 5. / 3.; + static constexpr double mean_molecular_weight = C::m_u; + static constexpr double boltzmann_constant = C::k_B; +}; + +template <> struct Physics_Traits { + // cell-centred + static constexpr bool is_hydro_enabled = true; + static constexpr int numMassScalars = 0; // number of mass scalars + static constexpr int numPassiveScalars = numMassScalars + 0; // number of passive scalars + static constexpr bool is_radiation_enabled = false; + // face-centred + static constexpr bool is_mhd_enabled = true; + static constexpr int nGroups = 1; // number of radiation groups +}; + +constexpr double rho0 = 1.0; // background density +constexpr double P0 = 1.0 / quokka::EOS_Traits::gamma; // background pressure +constexpr double v0 = 0.; // background velocity +constexpr double amp = 1.0e-6; // perturbation amplitude + +AMREX_GPU_DEVICE void computeWaveSolution(int i, int j, int k, amrex::Array4 const &state, amrex::GpuArray const &dx, + amrex::GpuArray const &prob_lo) +{ + const amrex::Real x_L = prob_lo[0] + (i + amrex::Real(0.0)) * dx[0]; + const amrex::Real x_R = prob_lo[0] + (i + amrex::Real(1.0)) * dx[0]; + const amrex::Real A = amp; + + const quokka::valarray R = {1.0, -1.0, 1.5}; // right eigenvector of sound wave + const quokka::valarray U_0 = {rho0, rho0 * v0, P0 / (quokka::EOS_Traits::gamma - 1.0) + 0.5 * rho0 * std::pow(v0, 2)}; + const quokka::valarray dU = (A * R / (2.0 * M_PI * dx[0])) * (std::cos(2.0 * M_PI * x_L) - std::cos(2.0 * M_PI * x_R)); + + double rho = U_0[0] + dU[0]; + double xmom = U_0[1] + dU[1]; + double Etot = U_0[2] + dU[2]; + double Eint = Etot - 0.5 * (xmom * xmom) / rho; + + state(i, j, k, HydroSystem::density_index) = rho; + state(i, j, k, HydroSystem::x1Momentum_index) = xmom; + state(i, j, k, HydroSystem::x2Momentum_index) = 0; + state(i, j, k, HydroSystem::x3Momentum_index) = 0; + state(i, j, k, HydroSystem::energy_index) = Etot; + state(i, j, k, HydroSystem::internalEnergy_index) = Eint; +} + +template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +{ + // extract grid information + amrex::GpuArray dx = grid_elem.dx_; + amrex::GpuArray prob_lo = grid_elem.prob_lo_; + const amrex::Array4 &state = grid_elem.array_; + const amrex::Box &indexRange = grid_elem.indexRange_; + + const int ncomp_cc = Physics_Indices::nvarTotal_cc; + // loop over the grid and set the initial condition + amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + for (int n = 0; n < ncomp_cc; ++n) { + state(i, j, k, n) = 0; // fill unused quantities with zeros + } + computeWaveSolution(i, j, k, state, dx, prob_lo); + }); +} + +template <> void RadhydroSimulation::setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) +{ + // extract grid information + const amrex::Array4 &state = grid_elem.array_; + const amrex::Box &indexRange = grid_elem.indexRange_; + const quokka::direction dir = grid_elem.dir_; + + if (dir == quokka::direction::x) { + amrex::ParallelFor( + indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 1.0 + (i % 2); }); + } else if (dir == quokka::direction::y) { + amrex::ParallelFor( + indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 2.0 + (j % 2); }); + } else if (dir == quokka::direction::z) { + amrex::ParallelFor( + indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 3.0 + (k % 2); }); + } +} + +void checkMFs(amrex::Vector> const &state1, + amrex::Vector> const &state2) +{ + double err = 0.0; + for (int level = 0; level < state1.size(); ++level) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + // initialise MF + const amrex::BoxArray &ba = state1[level][idim].boxArray(); + const amrex::DistributionMapping &dm = state1[level][idim].DistributionMap(); + int ncomp = state1[level][idim].nComp(); + int ngrow = state1[level][idim].nGrow(); + amrex::MultiFab mf_diff(ba, dm, ncomp, ngrow); + // compute difference between two MFs (at level) + amrex::MultiFab::Copy(mf_diff, state1[level][idim], 0, 0, ncomp, ngrow); + amrex::MultiFab::Subtract(mf_diff, state2[level][idim], 0, 0, ncomp, ngrow); + // compute error (summed over each component) + for (int icomp = 0; icomp < Physics_Indices::nvarPerDim_fc; ++icomp) { + err += mf_diff.norm1(icomp); + } + } + } + amrex::Print() << "Accumulated error in MFs read from chk-file: " << err << "\n"; + amrex::Print() << "\n"; + AMREX_ALWAYS_ASSERT(std::abs(err) == 0.0); +} + +auto problem_main() -> int +{ + // Problem initialization + const int ncomp_cc = Physics_Indices::nvarTotal_cc; + amrex::Vector BCs_cc(ncomp_cc); + for (int n = 0; n < ncomp_cc; ++n) { + for (int i = 0; i < AMREX_SPACEDIM; ++i) { + BCs_cc[n].setLo(i, amrex::BCType::int_dir); // periodic + BCs_cc[n].setHi(i, amrex::BCType::int_dir); + } + } + + const int nvars_fc = Physics_Indices::nvarTotal_fc; + amrex::Vector BCs_fc(nvars_fc); + for (int n = 0; n < nvars_fc; ++n) { + for (int i = 0; i < AMREX_SPACEDIM; ++i) { + BCs_fc[n].setLo(i, amrex::BCType::int_dir); // periodic + BCs_fc[n].setHi(i, amrex::BCType::int_dir); + } + } + + RadhydroSimulation sim_write(BCs_cc, BCs_fc); + sim_write.setInitialConditions(); + sim_write.evolve(); + amrex::Vector> const &state_new_fc_write = sim_write.getNewMF_fc(); + amrex::Print() << "\n"; + + RadhydroSimulation sim_restart(BCs_cc, BCs_fc); + sim_restart.setChkFile("chk00000"); + sim_restart.setInitialConditions(); + amrex::Vector> const &state_new_fc_restart = sim_restart.getNewMF_fc(); + amrex::Print() << "\n"; + + amrex::Print() << "Checking new FC MFs...\n"; + checkMFs(state_new_fc_write, state_new_fc_restart); + + return 0; +} diff --git a/src/AlfvenWave/test_alfven_wave.hpp b/src/AlfvenWave/test_alfven_wave.hpp new file mode 100644 index 000000000..d9ce50d47 --- /dev/null +++ b/src/AlfvenWave/test_alfven_wave.hpp @@ -0,0 +1,20 @@ +#ifndef TEST_ALFVEN_WAVE_HPP_ // NOLINT +#define TEST_ALFVEN_WAVE_HPP_ +//============================================================================== +// Copyright 2024 Neco Kriel. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file test_alfven_wave.hpp +/// \brief Defines a test problem to check alfven waves propogate correctly. +/// + +// external headers +#include + +// internal headers +#include "hydro_system.hpp" +#include "mhd_system.hpp" + +// function definitions + +#endif // TEST_ALFVEN_WAVE_HPP_ diff --git a/tests/alfven_wave.in b/tests/alfven_wave.in new file mode 100644 index 000000000..da4d0415c --- /dev/null +++ b/tests/alfven_wave.in @@ -0,0 +1,24 @@ +# ***************************************************************** +# Problem size and geometry +# ***************************************************************** +geometry.prob_lo = 0.0 0.0 0.0 +geometry.prob_hi = 1.0 1.0 1.0 +geometry.is_periodic = 1 1 1 + +# ***************************************************************** +# VERBOSITY +# ***************************************************************** +amr.v = 0 # verbosity in Amr + +# ***************************************************************** +# Resolution and refinement +# ***************************************************************** +amr.n_cell = 12 8 4 +amr.max_level = 0 # number of levels = max_level + 1 +amr.blocking_factor = 4 # grid size must be divisible by this + +do_reflux = 0 +do_subcycle = 0 + +plotfile_interval = 100 +checkpoint_interval = 200 \ No newline at end of file From df25a22cd52740c145d36dc430ec802d963d50db Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 6 Jun 2024 15:18:05 +1000 Subject: [PATCH 15/37] compute emf computes things correctly, but output still needs work --- src/mhd_system.hpp | 74 ++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index f73b3827e..9d04a9939 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -13,6 +13,7 @@ // library headers // internal headers +#include "AMReX_ParmParse.H" #include "AMReX_Print.H" #include "physics_info.hpp" #include "hydro_system.hpp" @@ -30,16 +31,14 @@ template class MHDSystem : public HyperbolicSystem::mhdFirstIndex, }; - static void SolveInductionEqn(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars_old, std::array &fcx_mf_cVars_new, amrex::GpuArray dx, int nghost_fc, double dt); + static void ComputeEMF(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, int nghost_fc); static void ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &reconstructRange, int reconstructionOrder); }; template -void MHDSystem::SolveInductionEqn(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars_old, std::array &fcx_mf_cVars_new, amrex::GpuArray dx, const int nghost_fc, const double dt) -{ - std::cout << "Computing EMF" << std::endl; - +void MHDSystem::ComputeEMF(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, const int nghost_fc) +{ // Loop over each box-array on the level // Note: all the different centerings still have the same distribution mapping, so it is fine for us to attach our looping to cc FArrayBox for (amrex::MFIter mfi(cc_mf_cVars); mfi.isValid(); ++mfi) { @@ -82,9 +81,9 @@ void MHDSystem::SolveInductionEqn(std::array fc_fabs_Bx_old = { - // FArrayBox(fcx_mf_cVars_old[0][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), - // FArrayBox(fcx_mf_cVars_old[1][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), - // FArrayBox(fcx_mf_cVars_old[2][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + // FArrayBox(fcx_mf_cVars[0][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + // FArrayBox(fcx_mf_cVars[1][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + // FArrayBox(fcx_mf_cVars[2][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), // }; // alternatively, make a complete copy of all the b-field data (+ghost cells) std::array fc_fabs_Bx_old; @@ -94,7 +93,7 @@ void MHDSystem::SolveInductionEqn(std::array::bfield_index); }); @@ -242,6 +241,18 @@ void MHDSystem::SolveInductionEqn(std::array::SolveInductionEqn(std::array::bfield_index + (wsolve + 2*iedge_rel2face + 2) % 3; - std::array idx = {0, 0, 0}; - idx[index_E2comp] = 1; - // compute the magnetic flux - const auto &fcxw_a4_rhs = fcx_mf_rhs[wsolve][mfi].array(); - amrex::ParallelFor(box_fcw, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - // the induction equation is written in an additive form: the RHS is evaluated in parts as each edge-centered electric field is computed - fcxw_a4_rhs(i,j,k) += (E2_ave(i,j,k) - E2_ave(i-idx[0],j-idx[1],k-idx[2])) / dx[wsolve]; - }); + // const int index_E2comp = MHDSystem::bfield_index + (wsolve + 2*iedge_rel2face + 2) % 3; + // std::array idx = {0, 0, 0}; + // idx[index_E2comp] = 1; + // // compute the magnetic flux + // const auto &fcxw_a4_rhs = fcx_mf_rhs[wsolve][mfi].array(); + // amrex::ParallelFor(box_fcw, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + // // the induction equation is written in an additive form: the RHS is evaluated in parts as each edge-centered electric field is computed + // fcxw_a4_rhs(i,j,k) += (E2_ave(i,j,k) - E2_ave(i-idx[0],j-idx[1],k-idx[2])) / dx[wsolve]; + // }); } - - const auto &fcxw_a4_rhs = fcx_mf_rhs[wsolve][mfi].const_array(); - const auto &fc_a4_Bx_old = fc_fabs_Bx_old[wsolve].const_array(); - auto fc_a4_Bx_new = fcx_mf_cVars_new[wsolve][mfi].array(); - amrex::ParallelFor(box_fcw, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - fc_a4_Bx_new(i, j, k) = fc_a4_Bx_old(i, j, k) + dt * fcxw_a4_rhs(i, j, k); - }); } } - - std::cout << "Done computing EMF" << std::endl; } template From 83af98ac43f463648ead5f0b3540e21e0e602a65 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 6 Jun 2024 15:18:54 +1000 Subject: [PATCH 16/37] propogate fast mhd wave speeds to compute emf func --- src/RadhydroSimulation.hpp | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 21eb6e93d..8b0192add 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -259,7 +259,7 @@ template class RadhydroSimulation : public AMRSimulation std::tuple, std::array>; auto computeHydroFluxes(amrex::MultiFab const &consVar, int nvars, int lev) - -> std::pair, std::array>; + -> std::tuple, std::array, std::array>; auto computeFOHydroFluxes(amrex::MultiFab const &consVar, int nvars, int lev) -> std::pair, std::array>; @@ -270,7 +270,7 @@ template class RadhydroSimulation : public AMRSimulation void hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, - amrex::MultiFab &x1FaceVel, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, + amrex::MultiFab &x1FaceVel, amrex::MultiFab &x1FSpds, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, int ng_reconstruct, int nvars); template @@ -1118,7 +1118,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto const &stateOld_fc = state_old_fc_tmp; auto &stateNew_fc = state_inter_fc_; - auto [fluxArrays, faceVel] = computeHydroFluxes(stateOld_cc, ncompHydro_, lev); + auto [fluxArrays, faceVel, fspds] = computeHydroFluxes(stateOld_cc, ncompHydro_, lev); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); @@ -1134,8 +1134,9 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); rhs4mhd_fc[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, nghost_fc_); + fspds[idim].FillBoundary(geom[lev].periodicity()); } - MHDSystem::SolveInductionEqn(rhs4mhd_fc, stateOld_cc, stateOld_fc, stateNew_fc, dx, nghost_fc_, dt_lev); + MHDSystem::ComputeEMF(rhs4mhd_fc, stateOld_cc, stateOld_fc, fspds, nghost_fc_); } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, faceVel, redoFlag); @@ -1237,7 +1238,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto const &stateOld_cc = state_old_cc_tmp; auto const &stateInter = state_inter_cc_; auto &stateFinal = state_new_cc_[lev]; - auto [fluxArrays, faceVel] = computeHydroFluxes(stateInter, ncompHydro_, lev); + auto [fluxArrays, faceVel, fspds] = computeHydroFluxes(stateInter, ncompHydro_, lev); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); @@ -1425,7 +1426,7 @@ auto RadhydroSimulation::expandFluxArrays(std::array auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) - -> std::pair, std::array> + -> std::tuple, std::array, std::array> { BL_PROFILE("RadhydroSimulation::computeHydroFluxes()"); @@ -1441,6 +1442,7 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co std::array facevel; std::array leftState; std::array rightState; + std::array fspds; for (int idim = 0; idim < 3; ++idim) { flatCoefs[idim] = amrex::MultiFab(ba_cc, dm, 1, flatteningGhost); @@ -1452,6 +1454,9 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co rightState[idim] = amrex::MultiFab(ba_fc, dm, nvars, reconstructGhost); flux[idim] = amrex::MultiFab(ba_fc, dm, nvars, 0); facevel[idim] = amrex::MultiFab(ba_fc, dm, 1, 0); + if constexpr (Physics_Traits::is_mhd_enabled) { + fspds[idim] = amrex::MultiFab(ba_fc, dm, 2, 1); + } } // conserved to primitive variables @@ -1463,11 +1468,11 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[2], flatteningGhost);) // compute flux functions - AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], flatCoefs[0], flatCoefs[1], flatCoefs[2], + AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], fspds[0], flatCoefs[0], flatCoefs[1], flatCoefs[2], reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], flatCoefs[0], flatCoefs[1], flatCoefs[2], + , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], fspds[1], flatCoefs[0], flatCoefs[1], flatCoefs[2], reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], flatCoefs[0], flatCoefs[1], flatCoefs[2], + , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], fspds[2], flatCoefs[0], flatCoefs[1], flatCoefs[2], reconstructGhost, nvars);) // synchronization point to prevent MultiFabs from going out of scope @@ -1509,13 +1514,13 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co } // return flux and face-centered velocities - return std::make_pair(std::move(flux), std::move(facevel)); + return std::make_tuple(std::move(flux), std::move(facevel), std::move(fspds)); } template template void RadhydroSimulation::hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, - amrex::MultiFab &flux, amrex::MultiFab &faceVel, amrex::MultiFab const &x1Flat, + amrex::MultiFab &flux, amrex::MultiFab &faceVel, amrex::MultiFab &x1FSpds, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, const int ng_reconstruct, const int nvars) { if (reconstructionOrder_ == 3) { @@ -1533,9 +1538,9 @@ void RadhydroSimulation::hydroFluxFunction(amrex::MultiFab const &pri // interface-centered kernel if constexpr (Physics_Traits::is_mhd_enabled) { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, &x1FSpds); } else { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, nullptr); } } @@ -1587,7 +1592,7 @@ void RadhydroSimulation::hydroFOFluxFunction(amrex::MultiFab const &p // donor-cell reconstruction HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); // LLF solver - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, nullptr); } template void RadhydroSimulation::swapRadiationState(amrex::MultiFab &stateOld_cc, amrex::MultiFab const &stateNew_cc) From 6afe3faead26dce11fe56c18a26629d5a0ed92f8 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 6 Jun 2024 15:19:20 +1000 Subject: [PATCH 17/37] remove debug evolve step --- src/FCQuantities/test_fc_quantities.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/FCQuantities/test_fc_quantities.cpp b/src/FCQuantities/test_fc_quantities.cpp index 2079ac9cd..8c21661c2 100644 --- a/src/FCQuantities/test_fc_quantities.cpp +++ b/src/FCQuantities/test_fc_quantities.cpp @@ -157,7 +157,6 @@ auto problem_main() -> int RadhydroSimulation sim_write(BCs_cc, BCs_fc); sim_write.setInitialConditions(); - sim_write.evolve(); amrex::Vector> const &state_new_fc_write = sim_write.getNewMF_fc(); amrex::Print() << "\n"; From 7552e650ba45b2c517a52268e47970d47814bd31 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 6 Jun 2024 15:19:29 +1000 Subject: [PATCH 18/37] add alfven wave test problem (not implemented yet) --- src/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b08290111..cd07be58a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -138,6 +138,9 @@ add_subdirectory(Advection) add_subdirectory(Advection2D) add_subdirectory(AdvectionSemiellipse) +add_subdirectory(FCQuantities) +add_subdirectory(AlfvenWave) + add_subdirectory(HydroBlast2D) add_subdirectory(HydroBlast3D) add_subdirectory(HydroContact) @@ -180,7 +183,6 @@ add_subdirectory(RadhydroPulseMG) add_subdirectory(BinaryOrbitCIC) add_subdirectory(Cooling) -add_subdirectory(FCQuantities) add_subdirectory(NSCBC) add_subdirectory(ODEIntegration) add_subdirectory(PassiveScalar) From 90b54175465f03d2d3d273fbac8b7598ed355d59 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 6 Jun 2024 15:20:19 +1000 Subject: [PATCH 19/37] output fast mhd wave speed --- src/HLLD.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/HLLD.hpp b/src/HLLD.hpp index 3c5b869e0..c894a3b7a 100644 --- a/src/HLLD.hpp +++ b/src/HLLD.hpp @@ -42,7 +42,7 @@ AMREX_FORCE_INLINE AMREX_GPU_DEVICE auto FastMagnetoSonicSpeed(double gamma, quo // HLLD solver following Miyoshi and Kusano (2005), hereafter MK5. template AMREX_FORCE_INLINE AMREX_GPU_DEVICE auto HLLD(quokka::HydroState const &sL, quokka::HydroState const &sR, - const double gamma, const double bx) -> quokka::valarray + const double gamma, const double bx) -> std::tuple, double, double> { //--- Step 1. Compute L/R states @@ -96,6 +96,8 @@ AMREX_FORCE_INLINE AMREX_GPU_DEVICE auto HLLD(quokka::HydroState F_hydro = {f_x.rho, f_x.mx, f_x.my, f_x.mz, f_x.E, 0.0}; - return F_hydro; + return std::make_tuple(std::move(F_hydro), fspd_m, fspd_p); } } // namespace quokka::Riemann From 1cba5a5c1a979ddadeae1fa2ee434571667675ec Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 6 Jun 2024 15:21:34 +1000 Subject: [PATCH 20/37] code to save mhd wavespeeds --- src/hydro_system.hpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 02089939c..665d9c7e9 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -16,6 +16,8 @@ #include "AMReX.H" #include "AMReX_Array4.H" #include "AMReX_BLassert.H" +#include "AMReX_MultiFabUtil.H" +#include "AMReX_Print.H" #include "AMReX_REAL.H" #include "AMReX_iMultiFab.H" @@ -116,7 +118,7 @@ template class HydroSystem : public HyperbolicSystem static void ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_mf, - amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, amrex::Real K_visc); + amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, amrex::Real K_visc, amrex::MultiFab* x1FSpds_mf = nullptr); template static void ComputeFirstOrderFluxes(amrex::Array4 const &consVar, array_t &x1FluxDiffusive, amrex::Box const &indexRange); @@ -888,7 +890,7 @@ template void HydroSystem::SyncDualEnergy(amrex: template template void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_mf, - amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, const amrex::Real K_visc) + amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, const amrex::Real K_visc, amrex::MultiFab* x1FSpds_mf) { // By convention, the interfaces are defined on the left edge of each @@ -903,12 +905,17 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu auto x1Flux_in = x1Flux_mf.arrays(); auto x1FaceVel_in = x1FaceVel_mf.arrays(); + amrex::MultiArray4 x1FSpds_in; + if constexpr (RIEMANN == RiemannSolver::HLLD) { + x1FSpds_in = (*x1FSpds_mf).arrays(); + } + amrex::ParallelFor(x1Flux_mf, [=] AMREX_GPU_DEVICE(int bx, int i_in, int j_in, int k_in) { quokka::Array4View x1LeftState(x1LeftState_in[bx]); quokka::Array4View x1RightState(x1RightState_in[bx]); + quokka::Array4View q(primVar_in[bx]); quokka::Array4View x1Flux(x1Flux_in[bx]); quokka::Array4View x1FaceVel(x1FaceVel_in[bx]); - quokka::Array4View q(primVar_in[bx]); auto [i, j, k] = quokka::reorderMultiIndex(i_in, j_in, k_in); @@ -1078,9 +1085,12 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu } else if constexpr (RIEMANN == RiemannSolver::LLF) { F_canonical = quokka::Riemann::LLF(sL, sR); } else if constexpr (RIEMANN == RiemannSolver::HLLD) { + quokka::Array4View x1FSpds(x1FSpds_in[bx]); // bx = 0 for testing purposes // TODO(Neco): pass correct bx value once magnetic fields are enabled - F_canonical = quokka::Riemann::HLLD(sL, sR, gamma_, 0.0); + auto [F_canonical, fspd_m, fspd_p] = quokka::Riemann::HLLD(sL, sR, gamma_, 0.0); + x1FSpds(i, j, k, 0) = fspd_m; + x1FSpds(i, j, k, 1) = fspd_p; } quokka::valarray F = F_canonical; From 40231d1573825ee6ab186c3d74e19e2af6b91f84 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 6 Jun 2024 16:02:03 +1000 Subject: [PATCH 21/37] return emf components --- src/RadhydroSimulation.hpp | 8 ++++---- src/mhd_system.hpp | 37 ++++++++++++++++++++----------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 8b0192add..6264fb183 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1130,13 +1130,13 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o redoFlag.setVal(quokka::redoFlag::none); if constexpr (Physics_Traits::is_mhd_enabled) { - std::array rhs4mhd_fc; + std::array, AMREX_SPACEDIM> emf_comps_ec; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); - rhs4mhd_fc[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, nghost_fc_); + emf_comps_ec[idim][0].define(amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim) + amrex::IntVect::TheDimensionVector((idim+1)%3)), dm, 1, nghost_fc_); + emf_comps_ec[idim][1].define(amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim) + amrex::IntVect::TheDimensionVector((idim+2)%3)), dm, 1, nghost_fc_); fspds[idim].FillBoundary(geom[lev].periodicity()); } - MHDSystem::ComputeEMF(rhs4mhd_fc, stateOld_cc, stateOld_fc, fspds, nghost_fc_); + MHDSystem::ComputeEMF(emf_comps_ec, stateOld_cc, stateOld_fc, fspds, nghost_fc_); } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, faceVel, redoFlag); diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index 9d04a9939..7acfb0989 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -31,13 +31,13 @@ template class MHDSystem : public HyperbolicSystem::mhdFirstIndex, }; - static void ComputeEMF(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, int nghost_fc); + static void ComputeEMF(std::array, AMREX_SPACEDIM> &ec_mf_emf_comps, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, int nghost_fc); static void ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &reconstructRange, int reconstructionOrder); }; template -void MHDSystem::ComputeEMF(std::array &fcx_mf_rhs, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, const int nghost_fc) +void MHDSystem::ComputeEMF(std::array, AMREX_SPACEDIM> &ec_mf_emf_comps, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, const int nghost_fc) { // Loop over each box-array on the level // Note: all the different centerings still have the same distribution mapping, so it is fine for us to attach our looping to cc FArrayBox @@ -47,15 +47,17 @@ void MHDSystem::ComputeEMF(std::array::ComputeEMF(std::array eci_fabs_E; + // // electric field on the cell-edge + // // indexing: field[2: i-edge on cell-face] + // std::array eci_fabs_E; + // to solve for the magnetic flux through each face we compute the line integral of the EMF around the cell-face. // so let's solve for the EMF along each of the two edges along the edge of the cell-face for (int iedge_rel2face = 0; iedge_rel2face < 2; ++iedge_rel2face) { @@ -124,8 +127,8 @@ void MHDSystem::ComputeEMF(std::arrayec const amrex::Box box_ecpgm1 = amrex::grow(box_ec, nghost_fc-1); - // define output electric field on the cell-edge - eci_fabs_E[iedge_rel2face].resize(box_ecpgm2, 1); + // // define output electric field on the cell-edge + // eci_fabs_E[iedge_rel2face].resize(box_ecpgm2, 1); // initialise FArrayBox for storing the edge-centered velocity fields averaged across the two extrapolation permutations // indexing: field[2: i-compnent][4: quadrant around edge] @@ -264,7 +267,7 @@ void MHDSystem::ComputeEMF(std::array Date: Fri, 7 Jun 2024 09:57:51 +1000 Subject: [PATCH 22/37] make a view of the bfield data / don't make an unnecessary copy --- src/mhd_system.hpp | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index 7acfb0989..ee5523494 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -80,26 +80,13 @@ void MHDSystem::ComputeEMF(std::array, cc_a4_Ux2(i,j,k) = px3 / rho; }); - // // indexing: field[3: x-component/x-face] - // // create a view of all the b-field data (+ghost cells; do not make another copy) - // std::array fc_fabs_Bx_old = { - // FArrayBox(fcx_mf_cVars[0][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), - // FArrayBox(fcx_mf_cVars[1][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), - // FArrayBox(fcx_mf_cVars[2][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), - // }; - // alternatively, make a complete copy of all the b-field data (+ghost cells) - std::array fc_fabs_Bx_old; - for (int windex = 0; windex < 3; ++windex) { - const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(windex); - const amrex::Box box_fcpg = amrex::convert(amrex::grow(box_cc, nghost_fc), ivec_cc2fc); - fc_fabs_Bx_old[windex].resize(box_fcpg, 1); - const auto &fc_a4_Bx = fc_fabs_Bx_old[windex].array(); - // extract face-centered magnetic fields - const auto &fc_a4_cVars_old = fcx_mf_cVars[windex][mfi].const_array(); - amrex::ParallelFor(box_fcpg, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - fc_a4_Bx(i, j, k) = fc_a4_cVars_old(i, j, k, MHDSystem::bfield_index); - }); - } + // indexing: field[3: x-component/x-face] + // create a view of all the b-field data (+ghost cells; do not make another copy) + std::array fc_fabs_Bx_old = { + amrex::FArrayBox(fcx_mf_cVars[0][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + amrex::FArrayBox(fcx_mf_cVars[1][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + amrex::FArrayBox(fcx_mf_cVars[2][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + }; // compute the magnetic flux through each cell-face for (int wsolve = 0; wsolve < 3; ++wsolve) { From 17fbf889f619128d0f87e250625f544a0ac3623d Mon Sep 17 00:00:00 2001 From: neco kriel Date: Fri, 7 Jun 2024 15:10:23 +1000 Subject: [PATCH 23/37] remove unused code --- src/mhd_system.hpp | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index ee5523494..1d58eba99 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -47,18 +47,6 @@ void MHDSystem::ComputeEMF(std::array, // In this function we distinguish between world (w:3), array (i:2), quandrant (q:4), and component (x:3) indexing with prefixes. We will use the x-prefix when the w- and i- indexes are the same. // We will minimise the storage footprint by only computing and holding onto the quantities required for calculating the EMF in the w-direction. This inadvertently leads to duplicate computation, but also significantly reduces the memory footprint, which is a bigger bottleneck. - // // initialise the rhs of the induction equation - // for (int windex = 0; windex < AMREX_SPACEDIM; ++windex) { - // const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(windex); - // const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); - // const auto &ec_a4_emf_x2 = ec_mf_emf_comps[windex][0][mfi].array(); - // const auto &ec_a4_emf_x3 = ec_mf_emf_comps[windex][1][mfi].array(); - // amrex::ParallelFor(box_fc, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - // ec_a4_emf_x2(i, j, k) = 0; - // ec_a4_emf_x3(i, j, k) = 0; - // }); - // } - // extract cell-centered velocity fields // indexing: field[3: x-component] std::array cc_fabs_Ux; @@ -264,16 +252,6 @@ void MHDSystem::ComputeEMF(std::array, fspd_x1(i,j,k,1) * fspd_x1(i,j,k,0) / (fspd_x1(i,j,k,1) + fspd_x1(i,j,k,0)) * (B0_p(i,j,k) - B0_m(i,j,k)) + fspd_x0(i,j,k,1) * fspd_x0(i,j,k,0) / (fspd_x0(i,j,k,1) + fspd_x0(i,j,k,0)) * (B1_p(i,j,k) - B1_m(i,j,k)); }); - - // const int index_E2comp = MHDSystem::bfield_index + (wsolve + 2*iedge_rel2face + 2) % 3; - // std::array idx = {0, 0, 0}; - // idx[index_E2comp] = 1; - // // compute the magnetic flux - // const auto &fcxw_a4_rhs = fcx_mf_rhs[wsolve][mfi].array(); - // amrex::ParallelFor(box_fcw, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - // // the induction equation is written in an additive form: the RHS is evaluated in parts as each edge-centered electric field is computed - // fcxw_a4_rhs(i,j,k) += (E2_ave(i,j,k) - E2_ave(i-idx[0],j-idx[1],k-idx[2])) / dx[wsolve]; - // }); } } } From c2dc7c4a3e37a013449facb96a38add6a8a8e103 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 05:14:57 +0000 Subject: [PATCH 24/37] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/AlfvenWave/test_alfven_wave.cpp | 20 +- src/HLLD.hpp | 4 +- src/RadhydroSimulation.hpp | 123 ++++--- src/hydro_system.hpp | 22 +- src/mhd_system.hpp | 510 ++++++++++++++-------------- 5 files changed, 355 insertions(+), 324 deletions(-) diff --git a/src/AlfvenWave/test_alfven_wave.cpp b/src/AlfvenWave/test_alfven_wave.cpp index 81d21928c..c12691657 100644 --- a/src/AlfvenWave/test_alfven_wave.cpp +++ b/src/AlfvenWave/test_alfven_wave.cpp @@ -42,10 +42,10 @@ template <> struct Physics_Traits { static constexpr int nGroups = 1; // number of radiation groups }; -constexpr double rho0 = 1.0; // background density +constexpr double rho0 = 1.0; // background density constexpr double P0 = 1.0 / quokka::EOS_Traits::gamma; // background pressure -constexpr double v0 = 0.; // background velocity -constexpr double amp = 1.0e-6; // perturbation amplitude +constexpr double v0 = 0.; // background velocity +constexpr double amp = 1.0e-6; // perturbation amplitude AMREX_GPU_DEVICE void computeWaveSolution(int i, int j, int k, amrex::Array4 const &state, amrex::GpuArray const &dx, amrex::GpuArray const &prob_lo) @@ -97,14 +97,14 @@ template <> void RadhydroSimulation::setInitialConditionsOnGridFaceV const quokka::direction dir = grid_elem.dir_; if (dir == quokka::direction::x) { - amrex::ParallelFor( - indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 1.0 + (i % 2); }); + amrex::ParallelFor(indexRange, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 1.0 + (i % 2); }); } else if (dir == quokka::direction::y) { - amrex::ParallelFor( - indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 2.0 + (j % 2); }); + amrex::ParallelFor(indexRange, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 2.0 + (j % 2); }); } else if (dir == quokka::direction::z) { - amrex::ParallelFor( - indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 3.0 + (k % 2); }); + amrex::ParallelFor(indexRange, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 3.0 + (k % 2); }); } } @@ -157,7 +157,7 @@ auto problem_main() -> int RadhydroSimulation sim_write(BCs_cc, BCs_fc); sim_write.setInitialConditions(); - sim_write.evolve(); + sim_write.evolve(); amrex::Vector> const &state_new_fc_write = sim_write.getNewMF_fc(); amrex::Print() << "\n"; diff --git a/src/HLLD.hpp b/src/HLLD.hpp index c894a3b7a..687dda168 100644 --- a/src/HLLD.hpp +++ b/src/HLLD.hpp @@ -96,8 +96,8 @@ AMREX_FORCE_INLINE AMREX_GPU_DEVICE auto HLLD(quokka::HydroState class RadhydroSimulation : public AMRSimulation::TracerPC; using AMRSimulation::nghost_cc_; - using AMRSimulation::nghost_fc_; + using AMRSimulation::nghost_fc_; using AMRSimulation::areInitialConditionsDefined_; using AMRSimulation::BCs_cc_; using AMRSimulation::BCs_fc_; @@ -125,7 +125,7 @@ template class RadhydroSimulation : public AMRSimulation::nvar_; // hydro static constexpr int ncompHyperbolic_ = RadSystem::nvarHyperbolic_; static constexpr int nstartHyperbolic_ = RadSystem::nstartHyperbolic_; - static constexpr int n_mhd_vars_per_dim_ = MHDSystem::nvar_per_dim_; // mhd + static constexpr int n_mhd_vars_per_dim_ = MHDSystem::nvar_per_dim_; // mhd amrex::Real radiationCflNumber_ = 0.3; int maxSubsteps_ = 10; // maximum number of radiation subcycles per hydro step @@ -229,8 +229,8 @@ template class RadhydroSimulation : public AMRSimulation &state_old_fc_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, int lev, - amrex::Real time, amrex::Real dt_lev) -> bool; + auto advanceHydroAtLevel(amrex::MultiFab &state_old_cc_tmp, std::array &state_old_fc_tmp, + amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, int lev, amrex::Real time, amrex::Real dt_lev) -> bool; void addStrangSplitSources(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev); auto addStrangSplitSourcesWithBuiltin(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev) -> bool; @@ -259,7 +259,8 @@ template class RadhydroSimulation : public AMRSimulation std::tuple, std::array>; auto computeHydroFluxes(amrex::MultiFab const &consVar, int nvars, int lev) - -> std::tuple, std::array, std::array>; + -> std::tuple, std::array, + std::array>; auto computeFOHydroFluxes(amrex::MultiFab const &consVar, int nvars, int lev) -> std::pair, std::array>; @@ -270,8 +271,8 @@ template class RadhydroSimulation : public AMRSimulation void hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, - amrex::MultiFab &x1FaceVel, amrex::MultiFab &x1FSpds, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, - int ng_reconstruct, int nvars); + amrex::MultiFab &x1FaceVel, amrex::MultiFab &x1FSpds, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, + amrex::MultiFab const &x3Flat, int ng_reconstruct, int nvars); template void hydroFOFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, @@ -931,13 +932,14 @@ void RadhydroSimulation::advanceHydroAtLevelWithRetries(int lev, amre amrex::MultiFab state_old_cc_tmp(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); amrex::Copy(state_old_cc_tmp, state_old_cc_[lev], 0, 0, Physics_Indices::nvarTotal_cc, nghost_cc_); - // create temporary array (different faces) of multifabs for old face-centered state - std::array state_old_fc_tmp; - if constexpr (Physics_Traits::is_mhd_enabled) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - state_old_fc_tmp[idim].define(amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), dmap[lev], Physics_Indices::nvarPerDim_fc, nghost_fc_); - amrex::Copy(state_old_fc_tmp[idim], state_old_fc_[lev][idim], 0, 0, Physics_Indices::nvarPerDim_fc, nghost_fc_); - } + // create temporary array (different faces) of multifabs for old face-centered state + std::array state_old_fc_tmp; + if constexpr (Physics_Traits::is_mhd_enabled) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + state_old_fc_tmp[idim].define(amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), dmap[lev], + Physics_Indices::nvarPerDim_fc, nghost_fc_); + amrex::Copy(state_old_fc_tmp[idim], state_old_fc_[lev][idim], 0, 0, Physics_Indices::nvarPerDim_fc, nghost_fc_); + } } // subcycle advanceHydroAtLevel, checking return value @@ -946,11 +948,12 @@ void RadhydroSimulation::advanceHydroAtLevelWithRetries(int lev, amre // since we are starting a new substep, we need to copy hydro state from // the new state vector to old state vector amrex::Copy(state_old_cc_tmp, state_new_cc_[lev], 0, 0, ncompHydro_, nghost_cc_); - if constexpr (Physics_Traits::is_mhd_enabled) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - amrex::Copy(state_old_fc_tmp[idim], state_new_fc_[lev][idim], 0, 0, Physics_Indices::nvarPerDim_fc, nghost_fc_); - } - } + if constexpr (Physics_Traits::is_mhd_enabled) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + amrex::Copy(state_old_fc_tmp[idim], state_new_fc_[lev][idim], 0, 0, Physics_Indices::nvarPerDim_fc, + nghost_fc_); + } + } } success = advanceHydroAtLevel(state_old_cc_tmp, state_old_fc_tmp, fr_as_crse, fr_as_fine, lev, time, dt_step); @@ -1036,8 +1039,9 @@ template void RadhydroSimulation::printCoordinat } template -auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_old_cc_tmp, std::array &state_old_fc_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, - int lev, amrex::Real time, amrex::Real dt_lev) -> bool +auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_old_cc_tmp, std::array &state_old_fc_tmp, + amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, int lev, amrex::Real time, + amrex::Real dt_lev) -> bool { BL_PROFILE("RadhydroSimulation::advanceHydroAtLevel()"); @@ -1048,7 +1052,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o fluxScaleFactor = 1.0; } - auto ba_cc = grids[lev]; + auto ba_cc = grids[lev]; auto dm = dmap[lev]; auto dx = geom[lev].CellSizeArray(); @@ -1063,14 +1067,14 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // create temporary multifab for intermediate state amrex::MultiFab state_inter_cc_(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); state_inter_cc_.setVal(0); // prevent assert in fillBoundaryConditions when radiation is enabled - std::array state_inter_fc_; - if constexpr (Physics_Traits::is_mhd_enabled) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); - state_inter_fc_[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, nghost_fc_); - state_inter_fc_[idim].setVal(0); - } - } + std::array state_inter_fc_; + if constexpr (Physics_Traits::is_mhd_enabled) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); + state_inter_fc_[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, nghost_fc_); + state_inter_fc_[idim].setVal(0); + } + } // create temporary multifabs for combined RK2 flux and time-average face velocity std::array flux_rk2; @@ -1115,8 +1119,8 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto const &stateOld_cc = state_old_cc_tmp; auto &stateNew_cc = state_inter_cc_; - auto const &stateOld_fc = state_old_fc_tmp; - auto &stateNew_fc = state_inter_fc_; + auto const &stateOld_fc = state_old_fc_tmp; + auto &stateNew_fc = state_inter_fc_; auto [fluxArrays, faceVel, fspds] = computeHydroFluxes(stateOld_cc, ncompHydro_, lev); @@ -1129,15 +1133,19 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::iMultiFab redoFlag(grids[lev], dmap[lev], 1, 1); redoFlag.setVal(quokka::redoFlag::none); - if constexpr (Physics_Traits::is_mhd_enabled) { - std::array, AMREX_SPACEDIM> emf_comps_ec; - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - emf_comps_ec[idim][0].define(amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim) + amrex::IntVect::TheDimensionVector((idim+1)%3)), dm, 1, nghost_fc_); - emf_comps_ec[idim][1].define(amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim) + amrex::IntVect::TheDimensionVector((idim+2)%3)), dm, 1, nghost_fc_); - fspds[idim].FillBoundary(geom[lev].periodicity()); - } - MHDSystem::ComputeEMF(emf_comps_ec, stateOld_cc, stateOld_fc, fspds, nghost_fc_); - } + if constexpr (Physics_Traits::is_mhd_enabled) { + std::array, AMREX_SPACEDIM> emf_comps_ec; + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + emf_comps_ec[idim][0].define( + amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim) + amrex::IntVect::TheDimensionVector((idim + 1) % 3)), dm, 1, + nghost_fc_); + emf_comps_ec[idim][1].define( + amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim) + amrex::IntVect::TheDimensionVector((idim + 2) % 3)), dm, 1, + nghost_fc_); + fspds[idim].FillBoundary(geom[lev].periodicity()); + } + MHDSystem::ComputeEMF(emf_comps_ec, stateOld_cc, stateOld_fc, fspds, nghost_fc_); + } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, faceVel, redoFlag); HydroSystem::PredictStep(stateOld_cc, stateNew_cc, rhs, dt_lev, ncompHydro_, redoFlag); @@ -1442,7 +1450,7 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co std::array facevel; std::array leftState; std::array rightState; - std::array fspds; + std::array fspds; for (int idim = 0; idim < 3; ++idim) { flatCoefs[idim] = amrex::MultiFab(ba_cc, dm, 1, flatteningGhost); @@ -1454,9 +1462,9 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co rightState[idim] = amrex::MultiFab(ba_fc, dm, nvars, reconstructGhost); flux[idim] = amrex::MultiFab(ba_fc, dm, nvars, 0); facevel[idim] = amrex::MultiFab(ba_fc, dm, 1, 0); - if constexpr (Physics_Traits::is_mhd_enabled) { - fspds[idim] = amrex::MultiFab(ba_fc, dm, 2, 1); - } + if constexpr (Physics_Traits::is_mhd_enabled) { + fspds[idim] = amrex::MultiFab(ba_fc, dm, 2, 1); + } } // conserved to primitive variables @@ -1468,12 +1476,12 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[2], flatteningGhost);) // compute flux functions - AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], fspds[0], flatCoefs[0], flatCoefs[1], flatCoefs[2], - reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], fspds[1], flatCoefs[0], flatCoefs[1], flatCoefs[2], - reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], fspds[2], flatCoefs[0], flatCoefs[1], flatCoefs[2], - reconstructGhost, nvars);) + AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], fspds[0], flatCoefs[0], flatCoefs[1], + flatCoefs[2], reconstructGhost, nvars); + , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], fspds[1], flatCoefs[0], flatCoefs[1], + flatCoefs[2], reconstructGhost, nvars); + , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], fspds[2], flatCoefs[0], flatCoefs[1], + flatCoefs[2], reconstructGhost, nvars);) // synchronization point to prevent MultiFabs from going out of scope amrex::Gpu::streamSynchronizeAll(); @@ -1538,9 +1546,11 @@ void RadhydroSimulation::hydroFluxFunction(amrex::MultiFab const &pri // interface-centered kernel if constexpr (Physics_Traits::is_mhd_enabled) { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, &x1FSpds); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, + &x1FSpds); } else { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, nullptr); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, + nullptr); } } @@ -1822,8 +1832,9 @@ void RadhydroSimulation::advanceRadiationMidpointRK2(int lev, amrex:: } template -void RadhydroSimulation::operatorSplitSourceTerms(amrex::Array4 const &stateNew_cc, const amrex::Box &indexRange, const amrex::Real time, - const double dt, const int stage, amrex::GpuArray const &dx, +void RadhydroSimulation::operatorSplitSourceTerms(amrex::Array4 const &stateNew_cc, const amrex::Box &indexRange, + const amrex::Real time, const double dt, const int stage, + amrex::GpuArray const &dx, amrex::GpuArray const &prob_lo, amrex::GpuArray const &prob_hi) { diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 665d9c7e9..fb00162a2 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -118,7 +118,8 @@ template class HydroSystem : public HyperbolicSystem static void ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_mf, - amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, amrex::Real K_visc, amrex::MultiFab* x1FSpds_mf = nullptr); + amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, amrex::Real K_visc, + amrex::MultiFab *x1FSpds_mf = nullptr); template static void ComputeFirstOrderFluxes(amrex::Array4 const &consVar, array_t &x1FluxDiffusive, amrex::Box const &indexRange); @@ -890,7 +891,8 @@ template void HydroSystem::SyncDualEnergy(amrex: template template void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_mf, - amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, const amrex::Real K_visc, amrex::MultiFab* x1FSpds_mf) + amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, const amrex::Real K_visc, + amrex::MultiFab *x1FSpds_mf) { // By convention, the interfaces are defined on the left edge of each @@ -905,10 +907,10 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu auto x1Flux_in = x1Flux_mf.arrays(); auto x1FaceVel_in = x1FaceVel_mf.arrays(); - amrex::MultiArray4 x1FSpds_in; - if constexpr (RIEMANN == RiemannSolver::HLLD) { - x1FSpds_in = (*x1FSpds_mf).arrays(); - } + amrex::MultiArray4 x1FSpds_in; + if constexpr (RIEMANN == RiemannSolver::HLLD) { + x1FSpds_in = (*x1FSpds_mf).arrays(); + } amrex::ParallelFor(x1Flux_mf, [=] AMREX_GPU_DEVICE(int bx, int i_in, int j_in, int k_in) { quokka::Array4View x1LeftState(x1LeftState_in[bx]); @@ -1085,12 +1087,12 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu } else if constexpr (RIEMANN == RiemannSolver::LLF) { F_canonical = quokka::Riemann::LLF(sL, sR); } else if constexpr (RIEMANN == RiemannSolver::HLLD) { - quokka::Array4View x1FSpds(x1FSpds_in[bx]); + quokka::Array4View x1FSpds(x1FSpds_in[bx]); // bx = 0 for testing purposes // TODO(Neco): pass correct bx value once magnetic fields are enabled - auto [F_canonical, fspd_m, fspd_p] = quokka::Riemann::HLLD(sL, sR, gamma_, 0.0); - x1FSpds(i, j, k, 0) = fspd_m; - x1FSpds(i, j, k, 1) = fspd_p; + auto [F_canonical, fspd_m, fspd_p] = quokka::Riemann::HLLD(sL, sR, gamma_, 0.0); + x1FSpds(i, j, k, 0) = fspd_m; + x1FSpds(i, j, k, 1) = fspd_p; } quokka::valarray F = F_canonical; diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index 1d58eba99..0f417af93 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -15,9 +15,9 @@ // internal headers #include "AMReX_ParmParse.H" #include "AMReX_Print.H" -#include "physics_info.hpp" #include "hydro_system.hpp" #include "hyperbolic_system.hpp" +#include "physics_info.hpp" #include "physics_numVars.hpp" /// Class for a MHD system of conservation laws @@ -31,275 +31,293 @@ template class MHDSystem : public HyperbolicSystem::mhdFirstIndex, }; - static void ComputeEMF(std::array, AMREX_SPACEDIM> &ec_mf_emf_comps, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, int nghost_fc); + static void ComputeEMF(std::array, AMREX_SPACEDIM> &ec_mf_emf_comps, amrex::MultiFab const &cc_mf_cVars, + std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, + int nghost_fc); - static void ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &reconstructRange, int reconstructionOrder); + static void ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &reconstructRange, + int reconstructionOrder); }; template -void MHDSystem::ComputeEMF(std::array, AMREX_SPACEDIM> &ec_mf_emf_comps, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, const int nghost_fc) -{ - // Loop over each box-array on the level - // Note: all the different centerings still have the same distribution mapping, so it is fine for us to attach our looping to cc FArrayBox - for (amrex::MFIter mfi(cc_mf_cVars); mfi.isValid(); ++mfi) { - const amrex::Box &box_cc = mfi.validbox(); +void MHDSystem::ComputeEMF(std::array, AMREX_SPACEDIM> &ec_mf_emf_comps, amrex::MultiFab const &cc_mf_cVars, + std::array const &fcx_mf_cVars, + std::array const &fcx_mf_fspds, const int nghost_fc) +{ + // Loop over each box-array on the level + // Note: all the different centerings still have the same distribution mapping, so it is fine for us to attach our looping to cc FArrayBox + for (amrex::MFIter mfi(cc_mf_cVars); mfi.isValid(); ++mfi) { + const amrex::Box &box_cc = mfi.validbox(); - // In this function we distinguish between world (w:3), array (i:2), quandrant (q:4), and component (x:3) indexing with prefixes. We will use the x-prefix when the w- and i- indexes are the same. - // We will minimise the storage footprint by only computing and holding onto the quantities required for calculating the EMF in the w-direction. This inadvertently leads to duplicate computation, but also significantly reduces the memory footprint, which is a bigger bottleneck. + // In this function we distinguish between world (w:3), array (i:2), quandrant (q:4), and component (x:3) indexing with prefixes. We will use + // the x-prefix when the w- and i- indexes are the same. We will minimise the storage footprint by only computing and holding onto the + // quantities required for calculating the EMF in the w-direction. This inadvertently leads to duplicate computation, but also significantly + // reduces the memory footprint, which is a bigger bottleneck. - // extract cell-centered velocity fields - // indexing: field[3: x-component] - std::array cc_fabs_Ux; - const amrex::Box &box_ccpg = amrex::grow(box_cc, nghost_fc); - cc_fabs_Ux[0].resize(box_ccpg, 1); - cc_fabs_Ux[1].resize(box_ccpg, 1); - cc_fabs_Ux[2].resize(box_ccpg, 1); - const auto &cc_a4_Ux0 = cc_fabs_Ux[0].array(); - const auto &cc_a4_Ux1 = cc_fabs_Ux[1].array(); - const auto &cc_a4_Ux2 = cc_fabs_Ux[2].array(); - const auto &cc_a4_cVars = cc_mf_cVars[mfi].const_array(); - amrex::ParallelFor(box_ccpg, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - const auto rho = cc_a4_cVars(i, j, k, HydroSystem::density_index); - const auto px1 = cc_a4_cVars(i, j, k, HydroSystem::x1Momentum_index); - const auto px2 = cc_a4_cVars(i, j, k, HydroSystem::x2Momentum_index); - const auto px3 = cc_a4_cVars(i, j, k, HydroSystem::x3Momentum_index); - cc_a4_Ux0(i,j,k) = px1 / rho; - cc_a4_Ux1(i,j,k) = px2 / rho; - cc_a4_Ux2(i,j,k) = px3 / rho; - }); + // extract cell-centered velocity fields + // indexing: field[3: x-component] + std::array cc_fabs_Ux; + const amrex::Box &box_ccpg = amrex::grow(box_cc, nghost_fc); + cc_fabs_Ux[0].resize(box_ccpg, 1); + cc_fabs_Ux[1].resize(box_ccpg, 1); + cc_fabs_Ux[2].resize(box_ccpg, 1); + const auto &cc_a4_Ux0 = cc_fabs_Ux[0].array(); + const auto &cc_a4_Ux1 = cc_fabs_Ux[1].array(); + const auto &cc_a4_Ux2 = cc_fabs_Ux[2].array(); + const auto &cc_a4_cVars = cc_mf_cVars[mfi].const_array(); + amrex::ParallelFor(box_ccpg, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + const auto rho = cc_a4_cVars(i, j, k, HydroSystem::density_index); + const auto px1 = cc_a4_cVars(i, j, k, HydroSystem::x1Momentum_index); + const auto px2 = cc_a4_cVars(i, j, k, HydroSystem::x2Momentum_index); + const auto px3 = cc_a4_cVars(i, j, k, HydroSystem::x3Momentum_index); + cc_a4_Ux0(i, j, k) = px1 / rho; + cc_a4_Ux1(i, j, k) = px2 / rho; + cc_a4_Ux2(i, j, k) = px3 / rho; + }); - // indexing: field[3: x-component/x-face] - // create a view of all the b-field data (+ghost cells; do not make another copy) - std::array fc_fabs_Bx_old = { - amrex::FArrayBox(fcx_mf_cVars[0][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), - amrex::FArrayBox(fcx_mf_cVars[1][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), - amrex::FArrayBox(fcx_mf_cVars[2][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), - }; + // indexing: field[3: x-component/x-face] + // create a view of all the b-field data (+ghost cells; do not make another copy) + std::array fc_fabs_Bx_old = { + amrex::FArrayBox(fcx_mf_cVars[0][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + amrex::FArrayBox(fcx_mf_cVars[1][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + amrex::FArrayBox(fcx_mf_cVars[2][mfi], amrex::make_alias, MHDSystem::bfield_index, 1), + }; - // compute the magnetic flux through each cell-face - for (int wsolve = 0; wsolve < 3; ++wsolve) { - const amrex::Box box_fcw = amrex::convert(box_cc, amrex::IntVect::TheDimensionVector(wsolve)); + // compute the magnetic flux through each cell-face + for (int wsolve = 0; wsolve < 3; ++wsolve) { + const amrex::Box box_fcw = amrex::convert(box_cc, amrex::IntVect::TheDimensionVector(wsolve)); - // // electric field on the cell-edge - // // indexing: field[2: i-edge on cell-face] - // std::array eci_fabs_E; + // // electric field on the cell-edge + // // indexing: field[2: i-edge on cell-face] + // std::array eci_fabs_E; - // to solve for the magnetic flux through each face we compute the line integral of the EMF around the cell-face. - // so let's solve for the EMF along each of the two edges along the edge of the cell-face - for (int iedge_rel2face = 0; iedge_rel2face < 2; ++iedge_rel2face) { - // for each of the two cell-edges on the cell-face + // to solve for the magnetic flux through each face we compute the line integral of the EMF around the cell-face. + // so let's solve for the EMF along each of the two edges along the edge of the cell-face + for (int iedge_rel2face = 0; iedge_rel2face < 2; ++iedge_rel2face) { + // for each of the two cell-edges on the cell-face - // define the two directions we need to extrapolate cell-centered velocity fields to get them to the cell-edge - // we will want to compute E2 = (U0 * B1 - U1 * B0) along the cell-edge - std::array w_extrap_dirs = { - wsolve, // dir-0: w-direction of the cell-face being solved for (relative to the cell-center) - (wsolve + iedge_rel2face + 1) % 3 // dir-1: w-direction of the cell-edge being solved for (relative to the cell-face) - }; - const amrex::IntVect ivec_cc2ec = amrex::IntVect::TheDimensionVector(w_extrap_dirs[0]) + amrex::IntVect::TheDimensionVector(w_extrap_dirs[1]); - const amrex::Box box_ec = amrex::convert(box_cc, ivec_cc2ec); - // you lose 2 ghost-cells when you reconstruct u-field cc->fc->ec - const amrex::Box box_ecpgm2 = amrex::grow(box_ec, nghost_fc-2); - // you lose 1 ghost-cell when you reconstruct b-field fc->ec - const amrex::Box box_ecpgm1 = amrex::grow(box_ec, nghost_fc-1); + // define the two directions we need to extrapolate cell-centered velocity fields to get them to the cell-edge + // we will want to compute E2 = (U0 * B1 - U1 * B0) along the cell-edge + std::array w_extrap_dirs = { + wsolve, // dir-0: w-direction of the cell-face being solved for (relative to the cell-center) + (wsolve + iedge_rel2face + 1) % 3 // dir-1: w-direction of the cell-edge being solved for (relative to the cell-face) + }; + const amrex::IntVect ivec_cc2ec = + amrex::IntVect::TheDimensionVector(w_extrap_dirs[0]) + amrex::IntVect::TheDimensionVector(w_extrap_dirs[1]); + const amrex::Box box_ec = amrex::convert(box_cc, ivec_cc2ec); + // you lose 2 ghost-cells when you reconstruct u-field cc->fc->ec + const amrex::Box box_ecpgm2 = amrex::grow(box_ec, nghost_fc - 2); + // you lose 1 ghost-cell when you reconstruct b-field fc->ec + const amrex::Box box_ecpgm1 = amrex::grow(box_ec, nghost_fc - 1); - // // define output electric field on the cell-edge - // eci_fabs_E[iedge_rel2face].resize(box_ecpgm2, 1); + // // define output electric field on the cell-edge + // eci_fabs_E[iedge_rel2face].resize(box_ecpgm2, 1); - // initialise FArrayBox for storing the edge-centered velocity fields averaged across the two extrapolation permutations - // indexing: field[2: i-compnent][4: quadrant around edge] - std::array, 2> ec_fabs_Ui_q; - // initialise temporary FArrayBox for storing the edge-centered velocity fields reconstructed from the cell-face - // indexing: field[2: i-side of edge] - std::array ec_fabs_U_ieside; - // define the four possible velocity field quantities that could be reconstructed at the cell-edge - // also define the temporary velocity field quantities that will be used for computing the extrapolation - ec_fabs_U_ieside[0].resize(box_ecpgm2, 1); - ec_fabs_U_ieside[1].resize(box_ecpgm2, 1); - // indexing: field[2: i-compnent][2: i-side of edge] - // note: magnetic field components cannot be discontinuous along themselves (i.e., either side of the face where they are stored) - std::array, 2> ec_fabs_Bi_ieside; - // define quantities - for (int icomp = 0; icomp < 2; ++icomp) { - ec_fabs_Bi_ieside[icomp][0].resize(box_ecpgm1, 1); - ec_fabs_Bi_ieside[icomp][1].resize(box_ecpgm1, 1); - for (int iquad = 0; iquad < 4; ++iquad) { - ec_fabs_Ui_q[icomp][iquad].resize(box_ecpgm2, 1); - } - } + // initialise FArrayBox for storing the edge-centered velocity fields averaged across the two extrapolation permutations + // indexing: field[2: i-compnent][4: quadrant around edge] + std::array, 2> ec_fabs_Ui_q; + // initialise temporary FArrayBox for storing the edge-centered velocity fields reconstructed from the cell-face + // indexing: field[2: i-side of edge] + std::array ec_fabs_U_ieside; + // define the four possible velocity field quantities that could be reconstructed at the cell-edge + // also define the temporary velocity field quantities that will be used for computing the extrapolation + ec_fabs_U_ieside[0].resize(box_ecpgm2, 1); + ec_fabs_U_ieside[1].resize(box_ecpgm2, 1); + // indexing: field[2: i-compnent][2: i-side of edge] + // note: magnetic field components cannot be discontinuous along themselves (i.e., either side of the face where they are + // stored) + std::array, 2> ec_fabs_Bi_ieside; + // define quantities + for (int icomp = 0; icomp < 2; ++icomp) { + ec_fabs_Bi_ieside[icomp][0].resize(box_ecpgm1, 1); + ec_fabs_Bi_ieside[icomp][1].resize(box_ecpgm1, 1); + for (int iquad = 0; iquad < 4; ++iquad) { + ec_fabs_Ui_q[icomp][iquad].resize(box_ecpgm2, 1); + } + } - // extrapolate the two required cell-centered velocity field components to the cell-edge - // there are two possible permutations for doing this, that is getting cell-centered quanties to a cell-edge - // first is cc->fc[dir-0]->ec and second is cc->fc[dir-1]->ec - for (int iperm = 0; iperm < 2; ++iperm) { - // for each permutation of extrapolating cc->ec + // extrapolate the two required cell-centered velocity field components to the cell-edge + // there are two possible permutations for doing this, that is getting cell-centered quanties to a cell-edge + // first is cc->fc[dir-0]->ec and second is cc->fc[dir-1]->ec + for (int iperm = 0; iperm < 2; ++iperm) { + // for each permutation of extrapolating cc->ec - // define quantities required for creating face-centered FArrayBox - const int w_extrap_dir2face = w_extrap_dirs[iperm]; - const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(w_extrap_dir2face); - const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); - const amrex::Box box_fcpgm1 = amrex::grow(box_fc, nghost_fc-1); - const auto dir2face = static_cast(w_extrap_dir2face); + // define quantities required for creating face-centered FArrayBox + const int w_extrap_dir2face = w_extrap_dirs[iperm]; + const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(w_extrap_dir2face); + const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); + const amrex::Box box_fcpgm1 = amrex::grow(box_fc, nghost_fc - 1); + const auto dir2face = static_cast(w_extrap_dir2face); - // define extrapolation direction to go from face to edge - const int w_extrap_dir2edge = w_extrap_dirs[(iperm+1) % 2]; - const auto dir2edge = static_cast(w_extrap_dir2edge); + // define extrapolation direction to go from face to edge + const int w_extrap_dir2edge = w_extrap_dirs[(iperm + 1) % 2]; + const auto dir2edge = static_cast(w_extrap_dir2edge); - // create temporary FArrayBox for storing the face-centered velocity fields reconstructed from the cell-center - // indexing: field[2: i-compnent][2: i-side of face] - std::array, 2> fc_fabs_Ui_ifside; - // extrapolate both required cell-centered velocity fields to the cell-edge - for (int icomp = 0; icomp < 2; ++icomp) { - const int w_comp = w_extrap_dirs[icomp]; - fc_fabs_Ui_ifside[icomp][0].resize(box_fcpgm1, 1); - fc_fabs_Ui_ifside[icomp][1].resize(box_fcpgm1, 1); - // extrapolate cell-centered velocity components to the cell-face - MHDSystem::ReconstructTo(dir2face, cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), fc_fabs_Ui_ifside[icomp][1].array(), box_fcpgm1, 1); - // extrapolate face-centered velocity components to the cell-edge - for (int iface = 0; iface < 2; ++iface) { - // reset values in temporary FArrayBox - ec_fabs_U_ieside[0].setVal(0.0); - ec_fabs_U_ieside[1].setVal(0.0); - // extrapolate face-centered velocity component to the cell-edge - MHDSystem::ReconstructTo(dir2edge, fc_fabs_Ui_ifside[icomp][iface].array(), ec_fabs_U_ieside[0].array(), ec_fabs_U_ieside[1].array(), box_ecpgm2, 1); - // figure out which quadrant of the cell-edge this extrapolated velocity component corresponds with - int iquad0 = -1; - int iquad1 = -1; - // note: quadrants are defined based on where the quantity sits relative to the edge (dir-0, dir-1): - // (-,+) | (+,+) - // 1 | 2 - // ------+------ - // 0 | 3 - // (-,-) | (+,-) - if (iperm == 0) { - iquad0 = (iface == 0) ? 0 : 3; - iquad1 = (iface == 0) ? 1 : 2; - } else { - iquad0 = (iface == 0) ? 0 : 1; - iquad1 = (iface == 0) ? 3 : 2; - } - ec_fabs_Ui_q[icomp][iquad0].atomicAdd(ec_fabs_U_ieside[0], 0, 0, 1); - ec_fabs_Ui_q[icomp][iquad1].atomicAdd(ec_fabs_U_ieside[1], 0, 0, 1); - } - } - } - // finish averaging the two different ways for extrapolating cc->ec velocity fields - for (int icomp = 0; icomp < 2; ++icomp) { - for (int iquad = 0; iquad < 4; ++iquad) { - ec_fabs_Ui_q[icomp][iquad].mult(0.5, 0, 1); - } - } + // create temporary FArrayBox for storing the face-centered velocity fields reconstructed from the cell-center + // indexing: field[2: i-compnent][2: i-side of face] + std::array, 2> fc_fabs_Ui_ifside; + // extrapolate both required cell-centered velocity fields to the cell-edge + for (int icomp = 0; icomp < 2; ++icomp) { + const int w_comp = w_extrap_dirs[icomp]; + fc_fabs_Ui_ifside[icomp][0].resize(box_fcpgm1, 1); + fc_fabs_Ui_ifside[icomp][1].resize(box_fcpgm1, 1); + // extrapolate cell-centered velocity components to the cell-face + MHDSystem::ReconstructTo(dir2face, cc_fabs_Ux[w_comp].array(), fc_fabs_Ui_ifside[icomp][0].array(), + fc_fabs_Ui_ifside[icomp][1].array(), box_fcpgm1, 1); + // extrapolate face-centered velocity components to the cell-edge + for (int iface = 0; iface < 2; ++iface) { + // reset values in temporary FArrayBox + ec_fabs_U_ieside[0].setVal(0.0); + ec_fabs_U_ieside[1].setVal(0.0); + // extrapolate face-centered velocity component to the cell-edge + MHDSystem::ReconstructTo(dir2edge, fc_fabs_Ui_ifside[icomp][iface].array(), + ec_fabs_U_ieside[0].array(), ec_fabs_U_ieside[1].array(), + box_ecpgm2, 1); + // figure out which quadrant of the cell-edge this extrapolated velocity component corresponds with + int iquad0 = -1; + int iquad1 = -1; + // note: quadrants are defined based on where the quantity sits relative to the edge (dir-0, dir-1): + // (-,+) | (+,+) + // 1 | 2 + // ------+------ + // 0 | 3 + // (-,-) | (+,-) + if (iperm == 0) { + iquad0 = (iface == 0) ? 0 : 3; + iquad1 = (iface == 0) ? 1 : 2; + } else { + iquad0 = (iface == 0) ? 0 : 1; + iquad1 = (iface == 0) ? 3 : 2; + } + ec_fabs_Ui_q[icomp][iquad0].atomicAdd(ec_fabs_U_ieside[0], 0, 0, 1); + ec_fabs_Ui_q[icomp][iquad1].atomicAdd(ec_fabs_U_ieside[1], 0, 0, 1); + } + } + } + // finish averaging the two different ways for extrapolating cc->ec velocity fields + for (int icomp = 0; icomp < 2; ++icomp) { + for (int iquad = 0; iquad < 4; ++iquad) { + ec_fabs_Ui_q[icomp][iquad].mult(0.5, 0, 1); + } + } - // extrapolate the two required face-centered magnetic field components to the cell-edge - for (int icomp = 0; icomp < 2; ++icomp) { - // define extrapolation direction to go from face to edge - const int w_comp = w_extrap_dirs[icomp]; - const int w_extrap_dir2edge = w_extrap_dirs[(icomp+1) % 2]; - const auto dir2edge = static_cast(w_extrap_dir2edge); - const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(w_extrap_dir2edge); - const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); - // extrapolate face-centered magnetic components to the cell-edge - MHDSystem::ReconstructTo(dir2edge, fc_fabs_Bx_old[w_comp].array(), ec_fabs_Bi_ieside[icomp][0].array(), ec_fabs_Bi_ieside[icomp][1].array(), box_ecpgm1, 1); - } + // extrapolate the two required face-centered magnetic field components to the cell-edge + for (int icomp = 0; icomp < 2; ++icomp) { + // define extrapolation direction to go from face to edge + const int w_comp = w_extrap_dirs[icomp]; + const int w_extrap_dir2edge = w_extrap_dirs[(icomp + 1) % 2]; + const auto dir2edge = static_cast(w_extrap_dir2edge); + const amrex::IntVect ivec_cc2fc = amrex::IntVect::TheDimensionVector(w_extrap_dir2edge); + const amrex::Box box_fc = amrex::convert(box_cc, ivec_cc2fc); + // extrapolate face-centered magnetic components to the cell-edge + MHDSystem::ReconstructTo(dir2edge, fc_fabs_Bx_old[w_comp].array(), ec_fabs_Bi_ieside[icomp][0].array(), + ec_fabs_Bi_ieside[icomp][1].array(), box_ecpgm1, 1); + } - // indexing: field[4: quadrant around edge] - std::array ec_fabs_E_q; - // compute the EMF along the cell-edge - for (int iquad = 0; iquad < 4; ++iquad) { - // define EMF FArrayBox - ec_fabs_E_q[iquad].resize(box_ecpgm2, 1); - // extract relevant velocity and magnetic field components - const auto &U0_qi = ec_fabs_Ui_q[0][iquad].const_array(); - const auto &U1_qi = ec_fabs_Ui_q[1][iquad].const_array(); - const auto &B0_qi = ec_fabs_Bi_ieside[0][(iquad == 0 || iquad == 3) ? 0 : 1].const_array(); - const auto &B1_qi = ec_fabs_Bi_ieside[1][(iquad < 2) ? 0 : 1].const_array(); - // compute electric field in the quadrant about the cell-edge: cross product between velocity and magnetic field in that quadrant - const auto &E2_qi = ec_fabs_E_q[iquad].array(); - amrex::ParallelFor(box_ecpgm2, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - E2_qi(i,j,k) = U0_qi(i,j,k) * B1_qi(i,j,k) - U1_qi(i,j,k) * B0_qi(i,j,k); - }); - } + // indexing: field[4: quadrant around edge] + std::array ec_fabs_E_q; + // compute the EMF along the cell-edge + for (int iquad = 0; iquad < 4; ++iquad) { + // define EMF FArrayBox + ec_fabs_E_q[iquad].resize(box_ecpgm2, 1); + // extract relevant velocity and magnetic field components + const auto &U0_qi = ec_fabs_Ui_q[0][iquad].const_array(); + const auto &U1_qi = ec_fabs_Ui_q[1][iquad].const_array(); + const auto &B0_qi = ec_fabs_Bi_ieside[0][(iquad == 0 || iquad == 3) ? 0 : 1].const_array(); + const auto &B1_qi = ec_fabs_Bi_ieside[1][(iquad < 2) ? 0 : 1].const_array(); + // compute electric field in the quadrant about the cell-edge: cross product between velocity and magnetic field in that + // quadrant + const auto &E2_qi = ec_fabs_E_q[iquad].array(); + amrex::ParallelFor(box_ecpgm2, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + E2_qi(i, j, k) = U0_qi(i, j, k) * B1_qi(i, j, k) - U1_qi(i, j, k) * B0_qi(i, j, k); + }); + } - // extract wavespeeds - int w0_comp = -1; - int w1_comp = -1; - if (std::abs(w_extrap_dirs[0] - w_extrap_dirs[1]) == 2) { - w0_comp = std::max(w_extrap_dirs[0], w_extrap_dirs[1]); - w1_comp = std::min(w_extrap_dirs[0], w_extrap_dirs[1]); - } else { - w0_comp = std::min(w_extrap_dirs[0], w_extrap_dirs[1]); - w1_comp = std::max(w_extrap_dirs[0], w_extrap_dirs[1]); - } - const auto &fspd_x0 = fcx_mf_fspds[w0_comp][mfi].const_array(); - const auto &fspd_x1 = fcx_mf_fspds[w1_comp][mfi].const_array(); - // extract both components of magnetic field either side of the cell-edge - const auto &B0_m = ec_fabs_Bi_ieside[0][0].const_array(); - const auto &B0_p = ec_fabs_Bi_ieside[0][1].const_array(); - const auto &B1_m = ec_fabs_Bi_ieside[1][0].const_array(); - const auto &B1_p = ec_fabs_Bi_ieside[1][1].const_array(); - // extract all four quadrants of the electric field about the cell-edge - const auto &E2_q0 = ec_fabs_E_q[0].const_array(); - const auto &E2_q1 = ec_fabs_E_q[1].const_array(); - const auto &E2_q2 = ec_fabs_E_q[2].const_array(); - const auto &E2_q3 = ec_fabs_E_q[3].const_array(); - // compute electric field on the cell-edge - const auto &E2_ave = ec_mf_emf_comps[wsolve][iedge_rel2face][mfi].array(); - amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - E2_ave(i,j,k) = \ - fspd_x0(i,j,k,1) * fspd_x1(i,j,k,1) * E2_q0(i,j,k) + - fspd_x0(i,j,k,0) * fspd_x1(i,j,k,1) * E2_q3(i,j,k) + - fspd_x0(i,j,k,0) * fspd_x1(i,j,k,0) * E2_q1(i,j,k) + - fspd_x0(i,j,k,1) * fspd_x1(i,j,k,0) * E2_q2(i,j,k) - - fspd_x1(i,j,k,1) * fspd_x1(i,j,k,0) / (fspd_x1(i,j,k,1) + fspd_x1(i,j,k,0)) * (B0_p(i,j,k) - B0_m(i,j,k)) + - fspd_x0(i,j,k,1) * fspd_x0(i,j,k,0) / (fspd_x0(i,j,k,1) + fspd_x0(i,j,k,0)) * (B1_p(i,j,k) - B1_m(i,j,k)); - }); - } - } - } + // extract wavespeeds + int w0_comp = -1; + int w1_comp = -1; + if (std::abs(w_extrap_dirs[0] - w_extrap_dirs[1]) == 2) { + w0_comp = std::max(w_extrap_dirs[0], w_extrap_dirs[1]); + w1_comp = std::min(w_extrap_dirs[0], w_extrap_dirs[1]); + } else { + w0_comp = std::min(w_extrap_dirs[0], w_extrap_dirs[1]); + w1_comp = std::max(w_extrap_dirs[0], w_extrap_dirs[1]); + } + const auto &fspd_x0 = fcx_mf_fspds[w0_comp][mfi].const_array(); + const auto &fspd_x1 = fcx_mf_fspds[w1_comp][mfi].const_array(); + // extract both components of magnetic field either side of the cell-edge + const auto &B0_m = ec_fabs_Bi_ieside[0][0].const_array(); + const auto &B0_p = ec_fabs_Bi_ieside[0][1].const_array(); + const auto &B1_m = ec_fabs_Bi_ieside[1][0].const_array(); + const auto &B1_p = ec_fabs_Bi_ieside[1][1].const_array(); + // extract all four quadrants of the electric field about the cell-edge + const auto &E2_q0 = ec_fabs_E_q[0].const_array(); + const auto &E2_q1 = ec_fabs_E_q[1].const_array(); + const auto &E2_q2 = ec_fabs_E_q[2].const_array(); + const auto &E2_q3 = ec_fabs_E_q[3].const_array(); + // compute electric field on the cell-edge + const auto &E2_ave = ec_mf_emf_comps[wsolve][iedge_rel2face][mfi].array(); + amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + E2_ave(i, j, k) = fspd_x0(i, j, k, 1) * fspd_x1(i, j, k, 1) * E2_q0(i, j, k) + + fspd_x0(i, j, k, 0) * fspd_x1(i, j, k, 1) * E2_q3(i, j, k) + + fspd_x0(i, j, k, 0) * fspd_x1(i, j, k, 0) * E2_q1(i, j, k) + + fspd_x0(i, j, k, 1) * fspd_x1(i, j, k, 0) * E2_q2(i, j, k) - + fspd_x1(i, j, k, 1) * fspd_x1(i, j, k, 0) / (fspd_x1(i, j, k, 1) + fspd_x1(i, j, k, 0)) * + (B0_p(i, j, k) - B0_m(i, j, k)) + + fspd_x0(i, j, k, 1) * fspd_x0(i, j, k, 0) / (fspd_x0(i, j, k, 1) + fspd_x0(i, j, k, 0)) * + (B1_p(i, j, k) - B1_m(i, j, k)); + }); + } + } + } } template -void MHDSystem::ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &reconstructRange, const int reconstructionOrder) +void MHDSystem::ReconstructTo(FluxDir dir, arrayconst_t &cState, array_t &lState, array_t &rState, const amrex::Box &reconstructRange, + const int reconstructionOrder) { - if (reconstructionOrder == 3) { - switch (dir) { - case FluxDir::X1: - MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); - break; - case FluxDir::X2: - MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); - break; - case FluxDir::X3: - MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); - break; - } - } else if (reconstructionOrder == 2) { - switch (dir) { - case FluxDir::X1: - MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); - break; - case FluxDir::X2: - MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); - break; - case FluxDir::X3: - MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, 1); - break; - } - } else if (reconstructionOrder == 1) { - switch (dir) { - case FluxDir::X1: - MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); - break; - case FluxDir::X2: - MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); - break; - case FluxDir::X3: - MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); - break; - } - } else { - amrex::Abort("Invalid reconstruction order specified!"); - } + if (reconstructionOrder == 3) { + switch (dir) { + case FluxDir::X1: + MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); + break; + case FluxDir::X2: + MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); + break; + case FluxDir::X3: + MHDSystem::template ReconstructStatesPPM(cState, lState, rState, reconstructRange, reconstructRange, 1); + break; + } + } else if (reconstructionOrder == 2) { + switch (dir) { + case FluxDir::X1: + MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, + 1); + break; + case FluxDir::X2: + MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, + 1); + break; + case FluxDir::X3: + MHDSystem::template ReconstructStatesPLM(cState, lState, rState, reconstructRange, + 1); + break; + } + } else if (reconstructionOrder == 1) { + switch (dir) { + case FluxDir::X1: + MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); + break; + case FluxDir::X2: + MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); + break; + case FluxDir::X3: + MHDSystem::template ReconstructStatesConstant(cState, lState, rState, reconstructRange, 1); + break; + } + } else { + amrex::Abort("Invalid reconstruction order specified!"); + } } - #endif // HYDRO_SYSTEM_HPP_ From bffd5b3783c5a458467848f7c19a66cd8d90875a Mon Sep 17 00:00:00 2001 From: neco kriel Date: Tue, 9 Jul 2024 09:00:17 +1000 Subject: [PATCH 25/37] store edge-centered emf components more compactly: data reconstructed with different permutations can live in the same multifab --- src/mhd_system.hpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/mhd_system.hpp b/src/mhd_system.hpp index 0f417af93..e99568554 100644 --- a/src/mhd_system.hpp +++ b/src/mhd_system.hpp @@ -31,7 +31,7 @@ template class MHDSystem : public HyperbolicSystem::mhdFirstIndex, }; - static void ComputeEMF(std::array, AMREX_SPACEDIM> &ec_mf_emf_comps, amrex::MultiFab const &cc_mf_cVars, + static void ComputeEMF(std::array &ec_mf_emf_comps, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, int nghost_fc); @@ -40,7 +40,7 @@ template class MHDSystem : public HyperbolicSystem -void MHDSystem::ComputeEMF(std::array, AMREX_SPACEDIM> &ec_mf_emf_comps, amrex::MultiFab const &cc_mf_cVars, +void MHDSystem::ComputeEMF(std::array &ec_mf_emf_comps, amrex::MultiFab const &cc_mf_cVars, std::array const &fcx_mf_cVars, std::array const &fcx_mf_fspds, const int nghost_fc) { @@ -87,10 +87,6 @@ void MHDSystem::ComputeEMF(std::array, for (int wsolve = 0; wsolve < 3; ++wsolve) { const amrex::Box box_fcw = amrex::convert(box_cc, amrex::IntVect::TheDimensionVector(wsolve)); - // // electric field on the cell-edge - // // indexing: field[2: i-edge on cell-face] - // std::array eci_fabs_E; - // to solve for the magnetic flux through each face we compute the line integral of the EMF around the cell-face. // so let's solve for the EMF along each of the two edges along the edge of the cell-face for (int iedge_rel2face = 0; iedge_rel2face < 2; ++iedge_rel2face) { @@ -256,9 +252,11 @@ void MHDSystem::ComputeEMF(std::array, const auto &E2_q2 = ec_fabs_E_q[2].const_array(); const auto &E2_q3 = ec_fabs_E_q[3].const_array(); // compute electric field on the cell-edge - const auto &E2_ave = ec_mf_emf_comps[wsolve][iedge_rel2face][mfi].array(); + const auto &E2_ave = ec_mf_emf_comps[w0_comp+w1_comp-1][mfi].array(); + const int i_perm = (wsolve == w0_comp) ? 0 : 1; amrex::ParallelFor(box_ec, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - E2_ave(i, j, k) = fspd_x0(i, j, k, 1) * fspd_x1(i, j, k, 1) * E2_q0(i, j, k) + + E2_ave(i,j,k,i_perm) = \ + fspd_x0(i, j, k, 1) * fspd_x1(i, j, k, 1) * E2_q0(i, j, k) + fspd_x0(i, j, k, 0) * fspd_x1(i, j, k, 1) * E2_q3(i, j, k) + fspd_x0(i, j, k, 0) * fspd_x1(i, j, k, 0) * E2_q1(i, j, k) + fspd_x0(i, j, k, 1) * fspd_x1(i, j, k, 0) * E2_q2(i, j, k) - From b81e07c95999bf57869696800f2099b39444cde9 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Tue, 9 Jul 2024 09:00:40 +1000 Subject: [PATCH 26/37] more meaningful variable names --- src/RadhydroSimulation.hpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index e2671f328..945894619 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1122,7 +1122,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto const &stateOld_fc = state_old_fc_tmp; auto &stateNew_fc = state_inter_fc_; - auto [fluxArrays, faceVel, fspds] = computeHydroFluxes(stateOld_cc, ncompHydro_, lev); + auto [fluxArrays, faceVel, fast_mhd_wavespeeds] = computeHydroFluxes(stateOld_cc, ncompHydro_, lev); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); @@ -1134,17 +1134,17 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o redoFlag.setVal(quokka::redoFlag::none); if constexpr (Physics_Traits::is_mhd_enabled) { - std::array, AMREX_SPACEDIM> emf_comps_ec; + std::array, AMREX_SPACEDIM> ec_emf_components; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - emf_comps_ec[idim][0].define( + ec_emf_components[idim][0].define( amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim) + amrex::IntVect::TheDimensionVector((idim + 1) % 3)), dm, 1, nghost_fc_); - emf_comps_ec[idim][1].define( + ec_emf_components[idim][1].define( amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim) + amrex::IntVect::TheDimensionVector((idim + 2) % 3)), dm, 1, nghost_fc_); - fspds[idim].FillBoundary(geom[lev].periodicity()); + fast_mhd_wavespeeds[idim].FillBoundary(geom[lev].periodicity()); } - MHDSystem::ComputeEMF(emf_comps_ec, stateOld_cc, stateOld_fc, fspds, nghost_fc_); + MHDSystem::ComputeEMF(ec_emf_components, stateOld_cc, stateOld_fc, fast_mhd_wavespeeds, nghost_fc_); } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, faceVel, redoFlag); @@ -1246,7 +1246,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto const &stateOld_cc = state_old_cc_tmp; auto const &stateInter = state_inter_cc_; auto &stateFinal = state_new_cc_[lev]; - auto [fluxArrays, faceVel, fspds] = computeHydroFluxes(stateInter, ncompHydro_, lev); + auto [fluxArrays, faceVel, fast_mhd_wavespeeds] = computeHydroFluxes(stateInter, ncompHydro_, lev); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); @@ -1450,7 +1450,7 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co std::array facevel; std::array leftState; std::array rightState; - std::array fspds; + std::array fast_mhd_wavespeeds; for (int idim = 0; idim < 3; ++idim) { flatCoefs[idim] = amrex::MultiFab(ba_cc, dm, 1, flatteningGhost); @@ -1463,7 +1463,7 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co flux[idim] = amrex::MultiFab(ba_fc, dm, nvars, 0); facevel[idim] = amrex::MultiFab(ba_fc, dm, 1, 0); if constexpr (Physics_Traits::is_mhd_enabled) { - fspds[idim] = amrex::MultiFab(ba_fc, dm, 2, 1); + fast_mhd_wavespeeds[idim] = amrex::MultiFab(ba_fc, dm, 2, 1); } } @@ -1476,11 +1476,11 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[2], flatteningGhost);) // compute flux functions - AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], fspds[0], flatCoefs[0], flatCoefs[1], + AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], fast_mhd_wavespeeds[0], flatCoefs[0], flatCoefs[1], flatCoefs[2], reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], fspds[1], flatCoefs[0], flatCoefs[1], + , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], fast_mhd_wavespeeds[1], flatCoefs[0], flatCoefs[1], flatCoefs[2], reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], fspds[2], flatCoefs[0], flatCoefs[1], + , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], fast_mhd_wavespeeds[2], flatCoefs[0], flatCoefs[1], flatCoefs[2], reconstructGhost, nvars);) // synchronization point to prevent MultiFabs from going out of scope @@ -1522,7 +1522,7 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co } // return flux and face-centered velocities - return std::make_tuple(std::move(flux), std::move(facevel), std::move(fspds)); + return std::make_tuple(std::move(flux), std::move(facevel), std::move(fast_mhd_wavespeeds)); } template From 6bfa86e2dcd94ff269cb69289a34b87ca5354629 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Tue, 9 Jul 2024 09:02:33 +1000 Subject: [PATCH 27/37] store edge-centered emf components more compactly: data reconstructed with different permutations can live in the same multifab --- src/RadhydroSimulation.hpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 945894619..98611a958 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1134,16 +1134,12 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o redoFlag.setVal(quokka::redoFlag::none); if constexpr (Physics_Traits::is_mhd_enabled) { - std::array, AMREX_SPACEDIM> ec_emf_components; - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - ec_emf_components[idim][0].define( - amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim) + amrex::IntVect::TheDimensionVector((idim + 1) % 3)), dm, 1, - nghost_fc_); - ec_emf_components[idim][1].define( - amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim) + amrex::IntVect::TheDimensionVector((idim + 2) % 3)), dm, 1, - nghost_fc_); - fast_mhd_wavespeeds[idim].FillBoundary(geom[lev].periodicity()); - } + std::array ec_emf_components; + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto ba_ec = amrex::convert(ba_cc, amrex::IntVect(1,1,1) - amrex::IntVect::TheDimensionVector(2-idim)); + ec_emf_components[idim].define(ba_ec, dm, 2, nghost_fc_); + fast_mhd_wavespeeds[idim].FillBoundary(geom[lev].periodicity()); + } MHDSystem::ComputeEMF(ec_emf_components, stateOld_cc, stateOld_fc, fast_mhd_wavespeeds, nghost_fc_); } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); From b49a1f1449d08a2591d9a568c078a267c64ccd48 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Wed, 10 Jul 2024 12:12:32 +1000 Subject: [PATCH 28/37] synchronise emf values that are on identical edges shared between multiple boxes --- src/RadhydroSimulation.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 98611a958..629ff9b32 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1141,6 +1141,9 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o fast_mhd_wavespeeds[idim].FillBoundary(geom[lev].periodicity()); } MHDSystem::ComputeEMF(ec_emf_components, stateOld_cc, stateOld_fc, fast_mhd_wavespeeds, nghost_fc_); + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + ec_emf_components[idim].WeightedSync(geom.periodicity()); + } } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, faceVel, redoFlag); From 23aaa22e35b11a49ac3db0abef04a60fac005892 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Wed, 10 Jul 2024 12:16:46 +1000 Subject: [PATCH 29/37] add mask for the synching emf components --- src/RadhydroSimulation.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 629ff9b32..f1e6ca565 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1142,7 +1142,8 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } MHDSystem::ComputeEMF(ec_emf_components, stateOld_cc, stateOld_fc, fast_mhd_wavespeeds, nghost_fc_); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - ec_emf_components[idim].WeightedSync(geom.periodicity()); + auto mask = ec_emf_components[idim].OverlapMask(geom[lev].periodicity()); + ec_emf_components[idim].WeightedSync(*mask, geom[lev].periodicity()); } } HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); From 5b05c4d389e36ab290cedea2a18bdc01064a912c Mon Sep 17 00:00:00 2001 From: neco kriel Date: Mon, 22 Jul 2024 12:26:20 +1000 Subject: [PATCH 30/37] rename IC funcs for consistency and to indicate centering --- src/Advection/test_advection.cpp | 2 +- src/Advection2D/test_advection2d.cpp | 2 +- .../test_advection_semiellipse.cpp | 2 +- src/AdvectionSimulation.hpp | 8 +- src/AlfvenWave/test_alfven_wave.cpp | 175 ++++++++++-------- src/BinaryOrbitCIC/binary_orbit.cpp | 2 +- src/Cooling/test_cooling.cpp | 2 +- src/FCQuantities/test_fc_quantities.cpp | 4 +- src/HydroBlast2D/test_hydro2d_blast.cpp | 2 +- src/HydroBlast3D/test_hydro3d_blast.cpp | 2 +- src/HydroContact/test_hydro_contact.cpp | 2 +- src/HydroHighMach/test_hydro_highmach.cpp | 2 +- src/HydroKelvinHelmholz/test_hydro2d_kh.cpp | 2 +- src/HydroLeblanc/test_hydro_leblanc.cpp | 2 +- src/HydroQuirk/test_quirk.cpp | 2 +- .../test_hydro2d_rm.cpp | 2 +- src/HydroSMS/test_hydro_sms.cpp | 2 +- src/HydroShocktube/test_hydro_shocktube.cpp | 2 +- .../test_hydro_shocktube_cma.cpp | 2 +- src/HydroShuOsher/test_hydro_shuosher.cpp | 2 +- src/HydroVacuum/test_hydro_vacuum.cpp | 2 +- src/HydroWave/test_hydro_wave.cpp | 2 +- src/NSCBC/channel.cpp | 2 +- src/NSCBC/vortex.cpp | 2 +- src/PassiveScalar/test_scalars.cpp | 2 +- src/PopIII/popiii.cpp | 2 +- src/PrimordialChem/test_primordial_chem.cpp | 2 +- src/RadBeam/test_radiation_beam.cpp | 2 +- src/RadForce/test_radiation_force.cpp | 2 +- src/RadMarshak/test_radiation_marshak.cpp | 2 +- .../test_radiation_marshak_asymptotic.cpp | 2 +- .../test_radiation_marshak_cgs.cpp | 2 +- .../test_radiation_matter_coupling.cpp | 2 +- .../test_radiation_matter_coupling_rsla.cpp | 2 +- src/RadPulse/test_radiation_pulse.cpp | 2 +- src/RadShadow/test_radiation_shadow.cpp | 2 +- src/RadStreaming/test_radiation_streaming.cpp | 2 +- src/RadSuOlson/test_radiation_SuOlson.cpp | 2 +- src/RadTophat/test_radiation_tophat.cpp | 2 +- src/RadTube/test_radiation_tube.cpp | 2 +- src/RadhydroPulse/test_radhydro_pulse.cpp | 4 +- .../test_radhydro_pulse_dyn.cpp | 4 +- .../test_radhydro_pulse_grey.cpp | 4 +- .../test_radhydro_pulse_MG.cpp | 4 +- src/RadhydroShell/test_radhydro_shell.cpp | 2 +- src/RadhydroShock/test_radhydro_shock.cpp | 2 +- .../test_radhydro_shock_cgs.cpp | 2 +- .../test_radhydro_shock_multigroup.cpp | 2 +- src/RadhydroSimulation.hpp | 8 +- .../test_radhydro_uniform_advecting.cpp | 2 +- src/RayleighTaylor2D/test_hydro2d_rt.cpp | 2 +- src/RayleighTaylor3D/test_hydro3d_rt.cpp | 2 +- src/ShockCloud/cloud.cpp | 2 +- src/SphericalCollapse/spherical_collapse.cpp | 2 +- src/StarCluster/star_cluster.cpp | 2 +- src/simulation.hpp | 9 +- 56 files changed, 163 insertions(+), 151 deletions(-) diff --git a/src/Advection/test_advection.cpp b/src/Advection/test_advection.cpp index 8e1b837b0..47ad3f864 100644 --- a/src/Advection/test_advection.cpp +++ b/src/Advection/test_advection.cpp @@ -47,7 +47,7 @@ AMREX_GPU_DEVICE void ComputeExactSolution(int i, int j, int k, int n, amrex::Ar exact_arr(i, j, k, n) = value; } -template <> void AdvectionSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void AdvectionSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/Advection2D/test_advection2d.cpp b/src/Advection2D/test_advection2d.cpp index 7a59d5529..607febdf6 100644 --- a/src/Advection2D/test_advection2d.cpp +++ b/src/Advection2D/test_advection2d.cpp @@ -57,7 +57,7 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto exactSolutionAtIndex(int i, int j, amre return rho; } -template <> void AdvectionSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void AdvectionSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/AdvectionSemiellipse/test_advection_semiellipse.cpp b/src/AdvectionSemiellipse/test_advection_semiellipse.cpp index a1ae41dc1..e6e11cab0 100644 --- a/src/AdvectionSemiellipse/test_advection_semiellipse.cpp +++ b/src/AdvectionSemiellipse/test_advection_semiellipse.cpp @@ -46,7 +46,7 @@ AMREX_GPU_DEVICE void ComputeExactSolution(int i, int j, int k, int n, amrex::Ar exact_arr(i, j, k, n) = dens; } -template <> void AdvectionSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void AdvectionSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/AdvectionSimulation.hpp b/src/AdvectionSimulation.hpp index 905849cbc..c03536dc1 100644 --- a/src/AdvectionSimulation.hpp +++ b/src/AdvectionSimulation.hpp @@ -63,8 +63,8 @@ template class AdvectionSimulation : public AMRSimulation

amrex::Real override; void preCalculateInitialConditions() override; - void setInitialConditionsOnGrid(quokka::grid grid_elem) override; - void setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) override; + void setInitialConditionsOnGrid_cc(quokka::grid grid_elem) override; + void setInitialConditionsOnGrid_fc(quokka::grid grid_elem) override; void createInitialParticles() override; void advanceSingleTimestepAtLevel(int lev, amrex::Real time, amrex::Real dt_lev, int /*ncycle*/) override; void computeAfterTimestep() override; @@ -136,13 +136,13 @@ template void AdvectionSimulation::preCalculateI // user should implement using problem-specific template specialization } -template void AdvectionSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template void AdvectionSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // default empty implementation // user should implement using problem-specific template specialization } -template void AdvectionSimulation::setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) +template void AdvectionSimulation::setInitialConditionsOnGrid_fc(quokka::grid grid_elem) { // default empty implementation // user should implement using problem-specific template specialization diff --git a/src/AlfvenWave/test_alfven_wave.cpp b/src/AlfvenWave/test_alfven_wave.cpp index c12691657..7da437132 100644 --- a/src/AlfvenWave/test_alfven_wave.cpp +++ b/src/AlfvenWave/test_alfven_wave.cpp @@ -7,6 +7,7 @@ /// #include +#include #include #include #include @@ -39,104 +40,125 @@ template <> struct Physics_Traits { static constexpr bool is_radiation_enabled = false; // face-centred static constexpr bool is_mhd_enabled = true; - static constexpr int nGroups = 1; // number of radiation groups + static constexpr int nGroups = 1; // number of radiation groups // TODO(Neco): shold this be zero? }; -constexpr double rho0 = 1.0; // background density -constexpr double P0 = 1.0 / quokka::EOS_Traits::gamma; // background pressure -constexpr double v0 = 0.; // background velocity -constexpr double amp = 1.0e-6; // perturbation amplitude +// constants +constexpr double sound_speed = 1.0; +constexpr double gamma = 5. / 3.; -AMREX_GPU_DEVICE void computeWaveSolution(int i, int j, int k, amrex::Array4 const &state, amrex::GpuArray const &dx, - amrex::GpuArray const &prob_lo) +// we have set up the problem so that: +// the direction of wave propogation, vec(k), is aligned with the x1-direction +// the background magnetic field sits in the x1-x2 plane + +// background states +constexpr double bg_density = 1.0; +constexpr double bg_pressure = 1.0; +constexpr double bg_mag_amplitude = 1.0; + +// alignment of magnetic field with the direction of wave propogation (in the x1-x2 plane). recall that hat(k) = (1, 0, 0) and hat(delta_u) = (0, 1, 0) +constexpr double theta = 90.0; // degrees + +// wave amplitude: box length = 1, so |k| in [0, 1] +constexpr double k_amplitude = 0.5; + +// input perturbation: choose to do this via the relative denisty field in [0, 1]. remember, the linear regime is valid when this perturbation is small +constexpr double delta_b = 1e-8; + +AMREX_GPU_DEVICE void computeWaveSolution(int i, int j, int k, amrex::Array4 const &state, amrex::GpuArray const &dx, amrex::GpuArray const &prob_lo, quokka::centering cen, quokka::direction dir, double t) { - const amrex::Real x_L = prob_lo[0] + (i + amrex::Real(0.0)) * dx[0]; - const amrex::Real x_R = prob_lo[0] + (i + amrex::Real(1.0)) * dx[0]; - const amrex::Real A = amp; - - const quokka::valarray R = {1.0, -1.0, 1.5}; // right eigenvector of sound wave - const quokka::valarray U_0 = {rho0, rho0 * v0, P0 / (quokka::EOS_Traits::gamma - 1.0) + 0.5 * rho0 * std::pow(v0, 2)}; - const quokka::valarray dU = (A * R / (2.0 * M_PI * dx[0])) * (std::cos(2.0 * M_PI * x_L) - std::cos(2.0 * M_PI * x_R)); - - double rho = U_0[0] + dU[0]; - double xmom = U_0[1] + dU[1]; - double Etot = U_0[2] + dU[2]; - double Eint = Etot - 0.5 * (xmom * xmom) / rho; - - state(i, j, k, HydroSystem::density_index) = rho; - state(i, j, k, HydroSystem::x1Momentum_index) = xmom; - state(i, j, k, HydroSystem::x2Momentum_index) = 0; - state(i, j, k, HydroSystem::x3Momentum_index) = 0; - state(i, j, k, HydroSystem::energy_index) = Etot; - state(i, j, k, HydroSystem::internalEnergy_index) = Eint; + const amrex::Real x1_L = prob_lo[0]; + const amrex::Real x1_C = x1_L + (i + static_cast(0.5)) * dx[0]; + + const double cos_theta = std::cos(theta * 2.0 * M_PI); + const double sin_theta = std::sin(theta * 2.0 * M_PI); + + const double alfven_speed = bg_mag_amplitude / std::sqrt(bg_density); + const double bg_mag_x1 = bg_mag_amplitude * cos_theta; + const double bg_mag_x2 = bg_mag_amplitude * sin_theta; + const double bg_mag_x3 = 0.0; + + const double omega = std::sqrt(std::pow(alfven_speed,2) * std::pow(k_amplitude,2) * std::pow(cos_theta,2)); + + if (cen == quokka::centering::cc) { + const double cos_wave_C = std::cos(omega * t - k_amplitude * x1_C); + + const double density = bg_density; + const double pressure = bg_pressure; + const double x1vel = 0; + const double x2vel = 0; + const double x3vel = -omega * delta_b / (sound_speed * k_amplitude * cos_theta) * cos_wave_C; + const double x1mag = bg_mag_x1 * cos_theta; + const double x2mag = bg_mag_x2 * sin_theta; + const double x3mag = bg_mag_x3 * delta_b * cos_wave_C; + + const double Eint = pressure / (gamma - 1); + const double mom_tot = 0.5 * density * (std::pow(x1vel, 2) + std::pow(x2vel, 2) + std::pow(x3vel, 2)); + const double Ekin = 0.5 * std::pow(mom_tot,2) / density; + const double Emag = 0.5 * (std::pow(x1mag, 2) + std::pow(x2mag, 2) + std::pow(x3mag, 2)); + const double Etot = Ekin + Emag + Eint; + + state(i, j, k, HydroSystem::density_index) = density; + state(i, j, k, HydroSystem::x1Momentum_index) = x1vel * density; + state(i, j, k, HydroSystem::x2Momentum_index) = x2vel * density; + state(i, j, k, HydroSystem::x3Momentum_index) = x3vel * density; + state(i, j, k, HydroSystem::energy_index) = Etot; + state(i, j, k, HydroSystem::internalEnergy_index) = Eint; + } else if (cen == quokka::centering::fc) { + const double cos_wave_L = std::cos(omega * t - k_amplitude * x1_L); + + const double x1mag = bg_mag_x1 * cos_theta; + const double x2mag = bg_mag_x2 * sin_theta; + const double x3mag = bg_mag_x3 * delta_b * cos_wave_L; + + if (dir == quokka::direction::x) {state(i, j, k, MHDSystem::bfield_index) = x1mag;} + else if (dir == quokka::direction::y) {state(i, j, k, MHDSystem::bfield_index) = x2mag;} + else if (dir == quokka::direction::z) {state(i, j, k, MHDSystem::bfield_index) = x3mag;} + } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract grid information - amrex::GpuArray dx = grid_elem.dx_; - amrex::GpuArray prob_lo = grid_elem.prob_lo_; - const amrex::Array4 &state = grid_elem.array_; + const amrex::GpuArray dx = grid_elem.dx_; + const amrex::GpuArray prob_lo = grid_elem.prob_lo_; + const amrex::Array4 &state_cc = grid_elem.array_; const amrex::Box &indexRange = grid_elem.indexRange_; + const quokka::centering cen = grid_elem.cen_; + const quokka::direction dir = grid_elem.dir_; const int ncomp_cc = Physics_Indices::nvarTotal_cc; // loop over the grid and set the initial condition amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) { for (int n = 0; n < ncomp_cc; ++n) { - state(i, j, k, n) = 0; // fill unused quantities with zeros + state_cc(i, j, k, n) = 0; // fill unused quantities with zeros } - computeWaveSolution(i, j, k, state, dx, prob_lo); + computeWaveSolution(i, j, k, state_cc, dx, prob_lo, cen, dir, 0); }); } -template <> void RadhydroSimulation::setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_fc(quokka::grid grid_elem) { // extract grid information - const amrex::Array4 &state = grid_elem.array_; + const amrex::GpuArray dx = grid_elem.dx_; + const amrex::GpuArray prob_lo = grid_elem.prob_lo_; + const amrex::Array4 &state_fc = grid_elem.array_; const amrex::Box &indexRange = grid_elem.indexRange_; - const quokka::direction dir = grid_elem.dir_; - - if (dir == quokka::direction::x) { - amrex::ParallelFor(indexRange, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 1.0 + (i % 2); }); - } else if (dir == quokka::direction::y) { - amrex::ParallelFor(indexRange, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 2.0 + (j % 2); }); - } else if (dir == quokka::direction::z) { - amrex::ParallelFor(indexRange, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { state(i, j, k, MHDSystem::bfield_index) = 3.0 + (k % 2); }); - } -} + const quokka::centering cen = grid_elem.cen_; + const quokka::direction dir = grid_elem.dir_; -void checkMFs(amrex::Vector> const &state1, - amrex::Vector> const &state2) -{ - double err = 0.0; - for (int level = 0; level < state1.size(); ++level) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - // initialise MF - const amrex::BoxArray &ba = state1[level][idim].boxArray(); - const amrex::DistributionMapping &dm = state1[level][idim].DistributionMap(); - int ncomp = state1[level][idim].nComp(); - int ngrow = state1[level][idim].nGrow(); - amrex::MultiFab mf_diff(ba, dm, ncomp, ngrow); - // compute difference between two MFs (at level) - amrex::MultiFab::Copy(mf_diff, state1[level][idim], 0, 0, ncomp, ngrow); - amrex::MultiFab::Subtract(mf_diff, state2[level][idim], 0, 0, ncomp, ngrow); - // compute error (summed over each component) - for (int icomp = 0; icomp < Physics_Indices::nvarPerDim_fc; ++icomp) { - err += mf_diff.norm1(icomp); - } + const int ncomp_fc = Physics_Indices::nvarPerDim_fc; + // loop over the grid and set the initial condition + amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + for (int n = 0; n < ncomp_fc; ++n) { + state_fc(i, j, k, n) = 0; // fill unused quantities with zeros } - } - amrex::Print() << "Accumulated error in MFs read from chk-file: " << err << "\n"; - amrex::Print() << "\n"; - AMREX_ALWAYS_ASSERT(std::abs(err) == 0.0); + computeWaveSolution(i, j, k, state_fc, dx, prob_lo, cen, dir, 0); + }); } auto problem_main() -> int { - // Problem initialization const int ncomp_cc = Physics_Indices::nvarTotal_cc; amrex::Vector BCs_cc(ncomp_cc); for (int n = 0; n < ncomp_cc; ++n) { @@ -158,17 +180,6 @@ auto problem_main() -> int RadhydroSimulation sim_write(BCs_cc, BCs_fc); sim_write.setInitialConditions(); sim_write.evolve(); - amrex::Vector> const &state_new_fc_write = sim_write.getNewMF_fc(); - amrex::Print() << "\n"; - - RadhydroSimulation sim_restart(BCs_cc, BCs_fc); - sim_restart.setChkFile("chk00000"); - sim_restart.setInitialConditions(); - amrex::Vector> const &state_new_fc_restart = sim_restart.getNewMF_fc(); - amrex::Print() << "\n"; - - amrex::Print() << "Checking new FC MFs...\n"; - checkMFs(state_new_fc_write, state_new_fc_restart); return 0; } diff --git a/src/BinaryOrbitCIC/binary_orbit.cpp b/src/BinaryOrbitCIC/binary_orbit.cpp index e7d6acfce..3b78f0eda 100644 --- a/src/BinaryOrbitCIC/binary_orbit.cpp +++ b/src/BinaryOrbitCIC/binary_orbit.cpp @@ -56,7 +56,7 @@ template <> struct SimulationData { std::vector dist{}; }; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { const amrex::Box &indexRange = grid_elem.indexRange_; const amrex::Array4 &state_cc = grid_elem.array_; diff --git a/src/Cooling/test_cooling.cpp b/src/Cooling/test_cooling.cpp index 8173ecd9e..4cbe345a3 100644 --- a/src/Cooling/test_cooling.cpp +++ b/src/Cooling/test_cooling.cpp @@ -81,7 +81,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions( amrex::Gpu::streamSynchronize(); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // set initial conditions amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/FCQuantities/test_fc_quantities.cpp b/src/FCQuantities/test_fc_quantities.cpp index 8c21661c2..323b0402c 100644 --- a/src/FCQuantities/test_fc_quantities.cpp +++ b/src/FCQuantities/test_fc_quantities.cpp @@ -71,7 +71,7 @@ AMREX_GPU_DEVICE void computeWaveSolution(int i, int j, int k, amrex::Array4::internalEnergy_index) = Eint; } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract grid information amrex::GpuArray dx = grid_elem.dx_; @@ -89,7 +89,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(qu }); } -template <> void RadhydroSimulation::setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_fc(quokka::grid grid_elem) { // extract grid information const amrex::Array4 &state = grid_elem.array_; diff --git a/src/HydroBlast2D/test_hydro2d_blast.cpp b/src/HydroBlast2D/test_hydro2d_blast.cpp index f159f7b55..f2ae66e39 100644 --- a/src/HydroBlast2D/test_hydro2d_blast.cpp +++ b/src/HydroBlast2D/test_hydro2d_blast.cpp @@ -42,7 +42,7 @@ template <> struct Physics_Traits { static constexpr int nGroups = 1; // number of radiation groups }; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroBlast3D/test_hydro3d_blast.cpp b/src/HydroBlast3D/test_hydro3d_blast.cpp index c0e1f101a..354d669b4 100644 --- a/src/HydroBlast3D/test_hydro3d_blast.cpp +++ b/src/HydroBlast3D/test_hydro3d_blast.cpp @@ -59,7 +59,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // initialize a Sedov test problem using parameters from // Richard Klein and J. Bolstad diff --git a/src/HydroContact/test_hydro_contact.cpp b/src/HydroContact/test_hydro_contact.cpp index e807f4f72..b75d3f80b 100644 --- a/src/HydroContact/test_hydro_contact.cpp +++ b/src/HydroContact/test_hydro_contact.cpp @@ -40,7 +40,7 @@ template <> struct Physics_Traits { constexpr double v_contact = 0.0; // contact wave velocity -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroHighMach/test_hydro_highmach.cpp b/src/HydroHighMach/test_hydro_highmach.cpp index e52487813..aba8a9968 100644 --- a/src/HydroHighMach/test_hydro_highmach.cpp +++ b/src/HydroHighMach/test_hydro_highmach.cpp @@ -45,7 +45,7 @@ template <> struct Physics_Traits { static constexpr int nGroups = 1; // number of radiation groups }; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroKelvinHelmholz/test_hydro2d_kh.cpp b/src/HydroKelvinHelmholz/test_hydro2d_kh.cpp index 6df2ebb41..ab213a507 100644 --- a/src/HydroKelvinHelmholz/test_hydro2d_kh.cpp +++ b/src/HydroKelvinHelmholz/test_hydro2d_kh.cpp @@ -43,7 +43,7 @@ template <> struct Physics_Traits { static constexpr int nGroups = 1; // number of radiation groups }; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroLeblanc/test_hydro_leblanc.cpp b/src/HydroLeblanc/test_hydro_leblanc.cpp index 22567f09c..79615774e 100644 --- a/src/HydroLeblanc/test_hydro_leblanc.cpp +++ b/src/HydroLeblanc/test_hydro_leblanc.cpp @@ -45,7 +45,7 @@ template <> struct Physics_Traits { static constexpr int nGroups = 1; // number of radiation groups }; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroQuirk/test_quirk.cpp b/src/HydroQuirk/test_quirk.cpp index e6b5c824b..44a773516 100644 --- a/src/HydroQuirk/test_quirk.cpp +++ b/src/HydroQuirk/test_quirk.cpp @@ -64,7 +64,7 @@ constexpr Real ur = -5.0; constexpr Real pr = 0.6; int ishock_g = 0; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroRichtmeyerMeshkov/test_hydro2d_rm.cpp b/src/HydroRichtmeyerMeshkov/test_hydro2d_rm.cpp index 23317ae62..7c43c50d2 100644 --- a/src/HydroRichtmeyerMeshkov/test_hydro2d_rm.cpp +++ b/src/HydroRichtmeyerMeshkov/test_hydro2d_rm.cpp @@ -99,7 +99,7 @@ template <> void RadhydroSimulation::computeAfterTimes } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroSMS/test_hydro_sms.cpp b/src/HydroSMS/test_hydro_sms.cpp index a529dfc13..f8c362c40 100644 --- a/src/HydroSMS/test_hydro_sms.cpp +++ b/src/HydroSMS/test_hydro_sms.cpp @@ -38,7 +38,7 @@ template <> struct Physics_Traits { static constexpr int nGroups = 1; // number of radiation groups }; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroShocktube/test_hydro_shocktube.cpp b/src/HydroShocktube/test_hydro_shocktube.cpp index ebf3cce8e..6b873adc9 100644 --- a/src/HydroShocktube/test_hydro_shocktube.cpp +++ b/src/HydroShocktube/test_hydro_shocktube.cpp @@ -49,7 +49,7 @@ constexpr amrex::Real P_L = 100.0; constexpr amrex::Real rho_R = 1.0; constexpr amrex::Real P_R = 1.0; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroShocktubeCMA/test_hydro_shocktube_cma.cpp b/src/HydroShocktubeCMA/test_hydro_shocktube_cma.cpp index 02f471cd8..3302d3e7d 100644 --- a/src/HydroShocktubeCMA/test_hydro_shocktube_cma.cpp +++ b/src/HydroShocktubeCMA/test_hydro_shocktube_cma.cpp @@ -54,7 +54,7 @@ constexpr amrex::Real P_L = 1.0; constexpr amrex::Real rho_R = 0.125; constexpr amrex::Real P_R = 0.1; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray const dx = grid_elem.dx_; diff --git a/src/HydroShuOsher/test_hydro_shuosher.cpp b/src/HydroShuOsher/test_hydro_shuosher.cpp index f9b02b15d..255d130f6 100644 --- a/src/HydroShuOsher/test_hydro_shuosher.cpp +++ b/src/HydroShuOsher/test_hydro_shuosher.cpp @@ -40,7 +40,7 @@ template <> struct Physics_Traits { static constexpr int nGroups = 1; // number of radiation groups }; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroVacuum/test_hydro_vacuum.cpp b/src/HydroVacuum/test_hydro_vacuum.cpp index 217b43e1c..76fa89f1f 100644 --- a/src/HydroVacuum/test_hydro_vacuum.cpp +++ b/src/HydroVacuum/test_hydro_vacuum.cpp @@ -42,7 +42,7 @@ template <> struct Physics_Traits { static constexpr int nGroups = 1; // number of radiation groups }; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/HydroWave/test_hydro_wave.cpp b/src/HydroWave/test_hydro_wave.cpp index 747285bde..be61f3254 100644 --- a/src/HydroWave/test_hydro_wave.cpp +++ b/src/HydroWave/test_hydro_wave.cpp @@ -67,7 +67,7 @@ AMREX_GPU_DEVICE void computeWaveSolution(int i, int j, int k, amrex::Array4::internalEnergy_index) = Eint; } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/NSCBC/channel.cpp b/src/NSCBC/channel.cpp index 2d4145d02..adc54b692 100644 --- a/src/NSCBC/channel.cpp +++ b/src/NSCBC/channel.cpp @@ -81,7 +81,7 @@ AMREX_GPU_MANAGED amrex::GpuArray::numPassiveScala }; // namespace #endif -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // set initial conditions const amrex::Box &indexRange = grid_elem.indexRange_; diff --git a/src/NSCBC/vortex.cpp b/src/NSCBC/vortex.cpp index 77e9186d3..00bdc6ce8 100644 --- a/src/NSCBC/vortex.cpp +++ b/src/NSCBC/vortex.cpp @@ -73,7 +73,7 @@ AMREX_GPU_MANAGED amrex::Real w0 = NAN; // NOLINT(cppcoreguidelines-av AMREX_GPU_MANAGED amrex::GpuArray::nscalars_> s0{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) }; // namespace -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // set initial conditions const amrex::Box &indexRange = grid_elem.indexRange_; diff --git a/src/PassiveScalar/test_scalars.cpp b/src/PassiveScalar/test_scalars.cpp index ef74be078..91ca567c8 100644 --- a/src/PassiveScalar/test_scalars.cpp +++ b/src/PassiveScalar/test_scalars.cpp @@ -42,7 +42,7 @@ template <> struct Physics_Traits { constexpr double v_contact = 2.0; // contact wave velocity -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 25e244a90..cd0f0fbf1 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -170,7 +170,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // set initial conditions amrex::GpuArray const dx = grid_elem.dx_; diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index 8e6409dd6..c228c6fa7 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -121,7 +121,7 @@ template <> void RadhydroSimulation::preCalculateInitialCond network_init(); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // set initial conditions const amrex::Box &indexRange = grid_elem.indexRange_; diff --git a/src/RadBeam/test_radiation_beam.cpp b/src/RadBeam/test_radiation_beam.cpp index ee1f36351..b83315390 100644 --- a/src/RadBeam/test_radiation_beam.cpp +++ b/src/RadBeam/test_radiation_beam.cpp @@ -213,7 +213,7 @@ AMRSimulation::setCustomBoundaryConditions(const amrex::IntVect &iv } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object const amrex::Box &indexRange = grid_elem.indexRange_; diff --git a/src/RadForce/test_radiation_force.cpp b/src/RadForce/test_radiation_force.cpp index f7fc0ba0a..685dc3545 100644 --- a/src/RadForce/test_radiation_force.cpp +++ b/src/RadForce/test_radiation_force.cpp @@ -138,7 +138,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions( amrex::Gpu::streamSynchronizeAll(); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/RadMarshak/test_radiation_marshak.cpp b/src/RadMarshak/test_radiation_marshak.cpp index b1996d4d3..2d76d9821 100644 --- a/src/RadMarshak/test_radiation_marshak.cpp +++ b/src/RadMarshak/test_radiation_marshak.cpp @@ -163,7 +163,7 @@ AMRSimulation::setCustomBoundaryConditions(const amrex::IntVect consVar(i, j, k, RadSystem::x3GasMomentum_index) = 0.; } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { const amrex::Box &indexRange = grid_elem.indexRange_; const amrex::Array4 &state_cc = grid_elem.array_; diff --git a/src/RadMarshakAsymptotic/test_radiation_marshak_asymptotic.cpp b/src/RadMarshakAsymptotic/test_radiation_marshak_asymptotic.cpp index 3c643fd4a..2e6af0b98 100644 --- a/src/RadMarshakAsymptotic/test_radiation_marshak_asymptotic.cpp +++ b/src/RadMarshakAsymptotic/test_radiation_marshak_asymptotic.cpp @@ -164,7 +164,7 @@ AMRSimulation::setCustomBoundaryConditions(const amrex::IntVe consVar(i, j, k, RadSystem::x3GasMomentum_index) = 0.; } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { const amrex::Box &indexRange = grid_elem.indexRange_; const amrex::Array4 &state_cc = grid_elem.array_; diff --git a/src/RadMarshakCGS/test_radiation_marshak_cgs.cpp b/src/RadMarshakCGS/test_radiation_marshak_cgs.cpp index 44a4a6849..c5f2715ed 100644 --- a/src/RadMarshakCGS/test_radiation_marshak_cgs.cpp +++ b/src/RadMarshakCGS/test_radiation_marshak_cgs.cpp @@ -179,7 +179,7 @@ AMRSimulation::setCustomBoundaryConditions(const amrex::IntVe consVar(i, j, k, RadSystem::x3GasMomentum_index) = 0.; } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { const amrex::Box &indexRange = grid_elem.indexRange_; const amrex::Array4 &state_cc = grid_elem.array_; diff --git a/src/RadMatterCoupling/test_radiation_matter_coupling.cpp b/src/RadMatterCoupling/test_radiation_matter_coupling.cpp index 2ff2040c2..31631572c 100644 --- a/src/RadMatterCoupling/test_radiation_matter_coupling.cpp +++ b/src/RadMatterCoupling/test_radiation_matter_coupling.cpp @@ -106,7 +106,7 @@ constexpr double Erad0 = 1.0e12; // erg cm^-3 constexpr double Egas0 = 1.0e2; // erg cm^-3 constexpr double rho0 = 1.0e-7; // g cm^-3 -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { const amrex::Box &indexRange = grid_elem.indexRange_; const amrex::Array4 &state_cc = grid_elem.array_; diff --git a/src/RadMatterCouplingRSLA/test_radiation_matter_coupling_rsla.cpp b/src/RadMatterCouplingRSLA/test_radiation_matter_coupling_rsla.cpp index fb0dfb169..734d0a995 100644 --- a/src/RadMatterCouplingRSLA/test_radiation_matter_coupling_rsla.cpp +++ b/src/RadMatterCouplingRSLA/test_radiation_matter_coupling_rsla.cpp @@ -108,7 +108,7 @@ constexpr double Erad0 = 1.0e12; // erg cm^-3 constexpr double Egas0 = 1.0e2; // erg cm^-3 constexpr double rho0 = 1.0e-7; // g cm^-3 -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { const amrex::Box &indexRange = grid_elem.indexRange_; const amrex::Array4 &state_cc = grid_elem.array_; diff --git a/src/RadPulse/test_radiation_pulse.cpp b/src/RadPulse/test_radiation_pulse.cpp index 514e05b30..9a704a5c4 100644 --- a/src/RadPulse/test_radiation_pulse.cpp +++ b/src/RadPulse/test_radiation_pulse.cpp @@ -94,7 +94,7 @@ AMREX_GPU_HOST_DEVICE auto RadSystem::ComputePlanckOpacityTempDeri return opacity_deriv; } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/RadShadow/test_radiation_shadow.cpp b/src/RadShadow/test_radiation_shadow.cpp index 58a5b5318..c5880ea5f 100644 --- a/src/RadShadow/test_radiation_shadow.cpp +++ b/src/RadShadow/test_radiation_shadow.cpp @@ -114,7 +114,7 @@ AMRSimulation::setCustomBoundaryConditions(const amrex::IntVect & } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/RadStreaming/test_radiation_streaming.cpp b/src/RadStreaming/test_radiation_streaming.cpp index fd4e9fd51..143501442 100644 --- a/src/RadStreaming/test_radiation_streaming.cpp +++ b/src/RadStreaming/test_radiation_streaming.cpp @@ -65,7 +65,7 @@ AMREX_GPU_HOST_DEVICE auto RadSystem::ComputeFluxMeanOpacity(c return ComputePlanckOpacity(0.0, 0.0); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { const amrex::Box &indexRange = grid_elem.indexRange_; const amrex::Array4 &state_cc = grid_elem.array_; diff --git a/src/RadSuOlson/test_radiation_SuOlson.cpp b/src/RadSuOlson/test_radiation_SuOlson.cpp index 221e14789..2beaecc51 100644 --- a/src/RadSuOlson/test_radiation_SuOlson.cpp +++ b/src/RadSuOlson/test_radiation_SuOlson.cpp @@ -142,7 +142,7 @@ void RadSystem::SetRadEnergySource(array_t &radEnergySource, amr }); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { const amrex::Box &indexRange = grid_elem.indexRange_; const amrex::Array4 &state_cc = grid_elem.array_; diff --git a/src/RadTophat/test_radiation_tophat.cpp b/src/RadTophat/test_radiation_tophat.cpp index 773c3896f..e3a1b8c8f 100644 --- a/src/RadTophat/test_radiation_tophat.cpp +++ b/src/RadTophat/test_radiation_tophat.cpp @@ -192,7 +192,7 @@ AMRSimulation::setCustomBoundaryConditions(const amrex::IntVect & } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/RadTube/test_radiation_tube.cpp b/src/RadTube/test_radiation_tube.cpp index dcc864644..4091788d2 100644 --- a/src/RadTube/test_radiation_tube.cpp +++ b/src/RadTube/test_radiation_tube.cpp @@ -133,7 +133,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions( amrex::Gpu::streamSynchronizeAll(); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/RadhydroPulse/test_radhydro_pulse.cpp b/src/RadhydroPulse/test_radhydro_pulse.cpp index f5a81548f..6b7a54a62 100644 --- a/src/RadhydroPulse/test_radhydro_pulse.cpp +++ b/src/RadhydroPulse/test_radhydro_pulse.cpp @@ -125,7 +125,7 @@ AMREX_GPU_HOST_DEVICE auto RadSystem::ComputeFluxMeanOpacity(co return ComputePlanckOpacity(rho, Tgas); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray const dx = grid_elem.dx_; @@ -156,7 +156,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(qu state_cc(i, j, k, RadSystem::x3GasMomentum_index) = 0.; }); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray const dx = grid_elem.dx_; diff --git a/src/RadhydroPulseDyn/test_radhydro_pulse_dyn.cpp b/src/RadhydroPulseDyn/test_radhydro_pulse_dyn.cpp index 4b5120611..bca67f3b8 100644 --- a/src/RadhydroPulseDyn/test_radhydro_pulse_dyn.cpp +++ b/src/RadhydroPulseDyn/test_radhydro_pulse_dyn.cpp @@ -126,7 +126,7 @@ AMREX_GPU_HOST_DEVICE auto RadSystem::ComputeFluxMeanOpacity(co return ComputePlanckOpacity(rho, Tgas); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray const dx = grid_elem.dx_; @@ -157,7 +157,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(qu state_cc(i, j, k, RadSystem::x3GasMomentum_index) = 0.; }); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray const dx = grid_elem.dx_; diff --git a/src/RadhydroPulseGrey/test_radhydro_pulse_grey.cpp b/src/RadhydroPulseGrey/test_radhydro_pulse_grey.cpp index 2acea0e47..d2fbf6cbc 100644 --- a/src/RadhydroPulseGrey/test_radhydro_pulse_grey.cpp +++ b/src/RadhydroPulseGrey/test_radhydro_pulse_grey.cpp @@ -128,7 +128,7 @@ AMREX_GPU_HOST_DEVICE auto RadSystem::ComputeFluxMeanOpacity(co return RadSystem::ComputeFluxMeanOpacity(rho, Tgas); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray const dx = grid_elem.dx_; @@ -159,7 +159,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(qu state_cc(i, j, k, RadSystem::x3GasMomentum_index) = 0.; }); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray const dx = grid_elem.dx_; diff --git a/src/RadhydroPulseMG/test_radhydro_pulse_MG.cpp b/src/RadhydroPulseMG/test_radhydro_pulse_MG.cpp index 163e0324a..5c9daddcb 100644 --- a/src/RadhydroPulseMG/test_radhydro_pulse_MG.cpp +++ b/src/RadhydroPulseMG/test_radhydro_pulse_MG.cpp @@ -223,7 +223,7 @@ template <> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE auto RadSystem void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray const dx = grid_elem.dx_; @@ -262,7 +262,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(qu state_cc(i, j, k, RadSystem::x3GasMomentum_index) = 0.; }); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray const dx = grid_elem.dx_; diff --git a/src/RadhydroShell/test_radhydro_shell.cpp b/src/RadhydroShell/test_radhydro_shell.cpp index 3d4f25d74..54a55842b 100644 --- a/src/RadhydroShell/test_radhydro_shell.cpp +++ b/src/RadhydroShell/test_radhydro_shell.cpp @@ -184,7 +184,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions amrex::Gpu::streamSynchronizeAll(); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/RadhydroShock/test_radhydro_shock.cpp b/src/RadhydroShock/test_radhydro_shock.cpp index 611050b9c..8a8f20620 100644 --- a/src/RadhydroShock/test_radhydro_shock.cpp +++ b/src/RadhydroShock/test_radhydro_shock.cpp @@ -166,7 +166,7 @@ AMRSimulation::setCustomBoundaryConditions(const amrex::IntVect &i } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/RadhydroShockCGS/test_radhydro_shock_cgs.cpp b/src/RadhydroShockCGS/test_radhydro_shock_cgs.cpp index c461ebb75..ba3e25f63 100644 --- a/src/RadhydroShockCGS/test_radhydro_shock_cgs.cpp +++ b/src/RadhydroShockCGS/test_radhydro_shock_cgs.cpp @@ -163,7 +163,7 @@ AMRSimulation::setCustomBoundaryConditions(const amrex::IntVect &i } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/RadhydroShockMultigroup/test_radhydro_shock_multigroup.cpp b/src/RadhydroShockMultigroup/test_radhydro_shock_multigroup.cpp index 446be45f8..138f9dbe0 100644 --- a/src/RadhydroShockMultigroup/test_radhydro_shock_multigroup.cpp +++ b/src/RadhydroShockMultigroup/test_radhydro_shock_multigroup.cpp @@ -165,7 +165,7 @@ AMRSimulation::setCustomBoundaryConditions(const amrex::IntVect &i } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray const dx = grid_elem.dx_; diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index f1e6ca565..61c139e92 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -176,8 +176,8 @@ template class RadhydroSimulation : public AMRSimulation amrex::Real override; void preCalculateInitialConditions() override; - void setInitialConditionsOnGrid(quokka::grid grid_elem) override; - void setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) override; + void setInitialConditionsOnGrid_cc(quokka::grid grid_elem) override; + void setInitialConditionsOnGrid_fc(quokka::grid grid_elem) override; void createInitialParticles() override; void advanceSingleTimestepAtLevel(int lev, amrex::Real time, amrex::Real dt_lev, int ncycle) override; void computeAfterTimestep() override; @@ -474,13 +474,13 @@ template void RadhydroSimulation::preCalculateIn // user should implement using problem-specific template specialization } -template void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // default empty implementation // user should implement using problem-specific template specialization } -template void RadhydroSimulation::setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) +template void RadhydroSimulation::setInitialConditionsOnGrid_fc(quokka::grid grid_elem) { // default empty implementation // user should implement using problem-specific template specialization diff --git a/src/RadhydroUniformAdvecting/test_radhydro_uniform_advecting.cpp b/src/RadhydroUniformAdvecting/test_radhydro_uniform_advecting.cpp index 97c369c69..473161d9f 100644 --- a/src/RadhydroUniformAdvecting/test_radhydro_uniform_advecting.cpp +++ b/src/RadhydroUniformAdvecting/test_radhydro_uniform_advecting.cpp @@ -91,7 +91,7 @@ AMREX_GPU_HOST_DEVICE auto RadSystem::ComputeFluxMeanOpacity(const return ComputePlanckOpacity(rho, Tgas); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object const amrex::Box &indexRange = grid_elem.indexRange_; diff --git a/src/RayleighTaylor2D/test_hydro2d_rt.cpp b/src/RayleighTaylor2D/test_hydro2d_rt.cpp index 286c26afa..a024a9dbf 100644 --- a/src/RayleighTaylor2D/test_hydro2d_rt.cpp +++ b/src/RayleighTaylor2D/test_hydro2d_rt.cpp @@ -43,7 +43,7 @@ amrex::Real constexpr g_x = 0; amrex::Real constexpr g_y = -0.1; amrex::Real constexpr g_z = 0; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/RayleighTaylor3D/test_hydro3d_rt.cpp b/src/RayleighTaylor3D/test_hydro3d_rt.cpp index 14a29d282..2b0447d14 100644 --- a/src/RayleighTaylor3D/test_hydro3d_rt.cpp +++ b/src/RayleighTaylor3D/test_hydro3d_rt.cpp @@ -44,7 +44,7 @@ amrex::Real constexpr g_x = 0; amrex::Real constexpr g_y = 0; amrex::Real constexpr g_z = -0.1; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // extract variables required from the geom object amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/ShockCloud/cloud.cpp b/src/ShockCloud/cloud.cpp index 524e5d5c9..7cbb864d3 100644 --- a/src/ShockCloud/cloud.cpp +++ b/src/ShockCloud/cloud.cpp @@ -91,7 +91,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() amrex::Gpu::streamSynchronize(); } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // set initial conditions amrex::GpuArray const dx = grid_elem.dx_; diff --git a/src/SphericalCollapse/spherical_collapse.cpp b/src/SphericalCollapse/spherical_collapse.cpp index c96d610d6..87bf83ab4 100644 --- a/src/SphericalCollapse/spherical_collapse.cpp +++ b/src/SphericalCollapse/spherical_collapse.cpp @@ -47,7 +47,7 @@ template <> struct Physics_Traits { static constexpr int nGroups = 1; // number of radiation groups }; -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // set initial conditions amrex::GpuArray dx = grid_elem.dx_; diff --git a/src/StarCluster/star_cluster.cpp b/src/StarCluster/star_cluster.cpp index 832593c2b..474fe3efb 100644 --- a/src/StarCluster/star_cluster.cpp +++ b/src/StarCluster/star_cluster.cpp @@ -115,7 +115,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions( } } -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +template <> void RadhydroSimulation::setInitialConditionsOnGrid_cc(quokka::grid grid_elem) { // set initial conditions amrex::GpuArray const dx = grid_elem.dx_; diff --git a/src/simulation.hpp b/src/simulation.hpp index a846d6ed3..54667b4ad 100644 --- a/src/simulation.hpp +++ b/src/simulation.hpp @@ -223,8 +223,8 @@ template class AMRSimulation : public amrex::AmrCore virtual auto computeExtraPhysicsTimestep(int lev) -> amrex::Real = 0; virtual void advanceSingleTimestepAtLevel(int lev, amrex::Real time, amrex::Real dt_lev, int ncycle) = 0; virtual void preCalculateInitialConditions() = 0; - virtual void setInitialConditionsOnGrid(quokka::grid grid_elem) = 0; - virtual void setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) = 0; + virtual void setInitialConditionsOnGrid_cc(quokka::grid grid_elem) = 0; + virtual void setInitialConditionsOnGrid_fc(quokka::grid grid_elem) = 0; virtual void createInitialParticles() = 0; virtual void computeAfterTimestep() = 0; virtual void computeAfterEvolve(amrex::Vector &initSumCons) = 0; @@ -361,6 +361,7 @@ template class AMRSimulation : public amrex::AmrCore amrex::Vector>> fillpatcher_; // Nghost = number of ghost cells for each array + // TODO(Neco): 5 for MHD and 4 for hydro. so should cc = fc = 5 if MHD? int nghost_cc_ = 4; // PPM needs nghost >= 3, PPM+flattening needs nghost >= 4 int nghost_fc_ = Physics_Traits::is_mhd_enabled ? 4 : 2; // 4 needed for MHD, otherwise only 2 for tracer particles amrex::Vector componentNames_cc_; @@ -1612,7 +1613,7 @@ template void AMRSimulation::setInitialCondition quokka::grid grid_elem(state_new_cc_[level].array(iter), iter.validbox(), geom[level].CellSizeArray(), geom[level].ProbLoArray(), geom[level].ProbHiArray(), quokka::centering::cc, quokka::direction::na); // set initial conditions defined by the user - setInitialConditionsOnGrid(grid_elem); + setInitialConditionsOnGrid_cc(grid_elem); } // check that the valid state_new_cc_[level] is properly filled AMREX_ALWAYS_ASSERT(!state_new_cc_[level].contains_nan(0, ncomp_cc)); @@ -1636,7 +1637,7 @@ template void AMRSimulation::setInitialCondition quokka::grid grid_elem(state_new_fc_[level][idim].array(iter), iter.validbox(), geom[level].CellSizeArray(), geom[level].ProbLoArray(), geom[level].ProbHiArray(), quokka::centering::fc, static_cast(idim)); // set initial conditions defined by the user - setInitialConditionsOnGridFaceVars(grid_elem); + setInitialConditionsOnGrid_fc(grid_elem); } // check that the valid state_new_fc_[level][idim] data is filled properly AMREX_ALWAYS_ASSERT(!state_new_fc_[level][idim].contains_nan(0, ncomp_per_dim_fc)); From 4dedeb11ee753a4dd8b64504ef56310250d29849 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 5 Sep 2024 13:19:26 +1000 Subject: [PATCH 31/37] compute momentum for ICs --- src/AlfvenWave/test_alfven_wave.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/AlfvenWave/test_alfven_wave.cpp b/src/AlfvenWave/test_alfven_wave.cpp index 7da437132..5c71922c0 100644 --- a/src/AlfvenWave/test_alfven_wave.cpp +++ b/src/AlfvenWave/test_alfven_wave.cpp @@ -70,8 +70,8 @@ AMREX_GPU_DEVICE void computeWaveSolution(int i, int j, int k, amrex::Array4(0.5)) * dx[0]; - const double cos_theta = std::cos(theta * 2.0 * M_PI); - const double sin_theta = std::sin(theta * 2.0 * M_PI); + const double cos_theta = std::cos(theta * M_PI / 180.0); + const double sin_theta = std::sin(theta * M_PI / 180.0); const double alfven_speed = bg_mag_amplitude / std::sqrt(bg_density); const double bg_mag_x1 = bg_mag_amplitude * cos_theta; @@ -93,8 +93,8 @@ AMREX_GPU_DEVICE void computeWaveSolution(int i, int j, int k, amrex::Array4 Date: Thu, 5 Sep 2024 13:20:10 +1000 Subject: [PATCH 32/37] rename vars for consistency --- src/RadhydroSimulation.hpp | 66 +++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 61c139e92..092c08857 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -258,7 +258,7 @@ template class RadhydroSimulation : public AMRSimulation dx) -> std::tuple, std::array>; - auto computeHydroFluxes(amrex::MultiFab const &consVar, int nvars, int lev) + auto computeHydroFluxes(amrex::MultiFab const &consVar, std::array const &consVar_fc, int nvars, int lev) -> std::tuple, std::array, std::array>; @@ -271,8 +271,8 @@ template class RadhydroSimulation : public AMRSimulation void hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, - amrex::MultiFab &x1FaceVel, amrex::MultiFab &x1FSpds, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, - amrex::MultiFab const &x3Flat, int ng_reconstruct, int nvars); + amrex::MultiFab &x1FaceVel, amrex::MultiFab &x1FSpds, std::array const &consVar_fc, + amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, int ng_reconstruct, int nvars); template void hydroFOFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, @@ -1122,7 +1122,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto const &stateOld_fc = state_old_fc_tmp; auto &stateNew_fc = state_inter_fc_; - auto [fluxArrays, faceVel, fast_mhd_wavespeeds] = computeHydroFluxes(stateOld_cc, ncompHydro_, lev); + auto [fluxArrays, faceVel, fast_mhd_wavespeeds] = computeHydroFluxes(stateOld_cc, stateOld_fc, ncompHydro_, lev); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); @@ -1244,9 +1244,14 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o AMREX_ASSERT(!state_inter_cc_.contains_nan()); // check ghost zones auto const &stateOld_cc = state_old_cc_tmp; - auto const &stateInter = state_inter_cc_; - auto &stateFinal = state_new_cc_[lev]; - auto [fluxArrays, faceVel, fast_mhd_wavespeeds] = computeHydroFluxes(stateInter, ncompHydro_, lev); + auto const &stateInter_cc = state_inter_cc_; + + auto const &stateOld_fc = state_old_fc_tmp; + auto const &stateInter_fc = state_inter_fc_; + + auto &stateFinal_cc = state_new_cc_[lev]; + + auto [fluxArrays, faceVel, fast_mhd_wavespeeds] = computeHydroFluxes(stateInter_cc, stateInter_fc, ncompHydro_, lev); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); @@ -1259,7 +1264,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, avgFaceVel, redoFlag); - HydroSystem::PredictStep(stateOld_cc, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); + HydroSystem::PredictStep(stateOld_cc, stateFinal_cc, rhs, dt_lev, ncompHydro_, redoFlag); // do first-order flux correction (FOFC) amrex::Gpu::streamSynchronizeAll(); // just in case @@ -1269,7 +1274,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Print() << "[FOFC-2] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; const amrex::IntVect cell_idx = redoFlag.maxIndex(0); printCoordinates(lev, cell_idx); - amrex::print_state(stateFinal, cell_idx); + amrex::print_state(stateFinal_cc, cell_idx); } // synchronize redoFlag across ranks @@ -1282,7 +1287,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // re-do RK update HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); HydroSystem::AddInternalEnergyPdV(rhs, stateOld_cc, dx, avgFaceVel, redoFlag); - HydroSystem::PredictStep(stateOld_cc, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); + HydroSystem::PredictStep(stateOld_cc, stateFinal_cc, rhs, dt_lev, ncompHydro_, redoFlag); amrex::Gpu::streamSynchronizeAll(); // just in case amrex::Long ncells_bad = redoFlag.sum(0); @@ -1293,7 +1298,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // print cell state amrex::Print() << "[FOFC-2] Flux correction failed:\n"; printCoordinates(lev, cell_idx); - amrex::print_state(stateFinal, cell_idx); + amrex::print_state(stateFinal_cc, cell_idx); amrex::Print() << "[FOFC-2] failed for " << ncells_bad << " cells on level " << lev << "\n"; } if (abortOnFofcFailure_ != 0) { @@ -1303,11 +1308,11 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } // prevent vacuum - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal_cc); if (useDualEnergy_ == 1) { // sync internal energy (requires positive density) - HydroSystem::SyncDualEnergy(stateFinal); + HydroSystem::SyncDualEnergy(stateFinal_cc); } if (do_reflux == 1) { @@ -1433,7 +1438,7 @@ auto RadhydroSimulation::expandFluxArrays(std::array -auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) +auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &consVar, std::array const &consVar_fc, const int nvars, const int lev) -> std::tuple, std::array, std::array> { BL_PROFILE("RadhydroSimulation::computeHydroFluxes()"); @@ -1476,12 +1481,9 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[2], flatteningGhost);) // compute flux functions - AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], fast_mhd_wavespeeds[0], flatCoefs[0], flatCoefs[1], - flatCoefs[2], reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], fast_mhd_wavespeeds[1], flatCoefs[0], flatCoefs[1], - flatCoefs[2], reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], fast_mhd_wavespeeds[2], flatCoefs[0], flatCoefs[1], - flatCoefs[2], reconstructGhost, nvars);) + AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], fast_mhd_wavespeeds[0], consVar_fc, flatCoefs[0], flatCoefs[1], flatCoefs[2], reconstructGhost, nvars); + , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], fast_mhd_wavespeeds[1], consVar_fc, flatCoefs[0], flatCoefs[1], flatCoefs[2], reconstructGhost, nvars); + , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], fast_mhd_wavespeeds[2], consVar_fc, flatCoefs[0], flatCoefs[1], flatCoefs[2], reconstructGhost, nvars);) // synchronization point to prevent MultiFabs from going out of scope amrex::Gpu::streamSynchronizeAll(); @@ -1528,8 +1530,8 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co template template void RadhydroSimulation::hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, - amrex::MultiFab &flux, amrex::MultiFab &faceVel, amrex::MultiFab &x1FSpds, amrex::MultiFab const &x1Flat, - amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, const int ng_reconstruct, const int nvars) + amrex::MultiFab &flux, amrex::MultiFab &faceVel, amrex::MultiFab &x1FSpds, std::array const &consVar_fc, + amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, const int ng_reconstruct, const int nvars) { if (reconstructionOrder_ == 3) { HyperbolicSystem::template ReconstructStatesPPM

(primVar, leftState, rightState, ng_reconstruct, nvars); @@ -1546,11 +1548,9 @@ void RadhydroSimulation::hydroFluxFunction(amrex::MultiFab const &pri // interface-centered kernel if constexpr (Physics_Traits::is_mhd_enabled) { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, - &x1FSpds); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, &x1FSpds, &consVar_fc); } else { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, - nullptr); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, nullptr, nullptr); } } @@ -1602,7 +1602,7 @@ void RadhydroSimulation::hydroFOFluxFunction(amrex::MultiFab const &p // donor-cell reconstruction HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); // LLF solver - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, nullptr); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, nullptr, nullptr); } template void RadhydroSimulation::swapRadiationState(amrex::MultiFab &stateOld_cc, amrex::MultiFab const &stateNew_cc) @@ -1743,13 +1743,13 @@ void RadhydroSimulation::advanceRadiationSubstepAtLevel(int lev, amre for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); auto const &stateOld_cc = state_old_cc_[lev].const_array(iter); - auto const &stateInter = state_new_cc_[lev].const_array(iter); + auto const &stateInter_cc = state_new_cc_[lev].const_array(iter); auto const &stateNew_cc = state_new_cc_[lev].array(iter); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter_cc, indexRange, ncompHyperbolic_, dx); // Stage 2 of RK2-SSP RadSystem::AddFluxesRK2( - stateNew_cc, stateOld_cc, stateInter, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + stateNew_cc, stateOld_cc, stateInter_cc, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, dt_radiation, dx, indexRange, ncompHyperbolic_); @@ -1809,14 +1809,14 @@ void RadhydroSimulation::advanceRadiationMidpointRK2(int lev, amrex:: for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); auto const &stateOld_cc = state_old_cc_[lev].const_array(iter); - auto const &stateInter = state_new_cc_[lev].const_array(iter); + auto const &stateInter_cc = state_new_cc_[lev].const_array(iter); auto const &stateNew_cc = state_new_cc_[lev].array(iter); auto [fluxArraysOld, fluxDiffusiveArraysOld] = computeRadiationFluxes(stateOld_cc, indexRange, ncompHyperbolic_, dx); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter_cc, indexRange, ncompHyperbolic_, dx); // Stage 2 of RK2-SSP RadSystem::AddFluxesRK2( - stateNew_cc, stateOld_cc, stateInter, {AMREX_D_DECL(fluxArraysOld[0].array(), fluxArraysOld[1].array(), fluxArraysOld[2].array())}, + stateNew_cc, stateOld_cc, stateInter_cc, {AMREX_D_DECL(fluxArraysOld[0].array(), fluxArraysOld[1].array(), fluxArraysOld[2].array())}, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, {AMREX_D_DECL(fluxDiffusiveArraysOld[0].const_array(), fluxDiffusiveArraysOld[1].const_array(), fluxDiffusiveArraysOld[2].const_array())}, {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, From 41622e7de3b238b7cb03c03b85a877666de82f7d Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 5 Sep 2024 13:21:28 +1000 Subject: [PATCH 33/37] (in-progress) setup of b-fields for Riemann solver --- src/hydro_system.hpp | 113 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 96 insertions(+), 17 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index fb00162a2..3c4b5677b 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -19,6 +19,7 @@ #include "AMReX_MultiFabUtil.H" #include "AMReX_Print.H" #include "AMReX_REAL.H" +#include "AMReX_SPACE.H" #include "AMReX_iMultiFab.H" // internal headers @@ -117,9 +118,9 @@ template class HydroSystem : public HyperbolicSystem - static void ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_mf, - amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, amrex::Real K_visc, - amrex::MultiFab *x1FSpds_mf = nullptr); + static void ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_cc_mf, + amrex::MultiFab const &x1RightState_cc_mf, amrex::MultiFab const &primVar_mf, amrex::Real K_visc, + amrex::MultiFab *x1FSpds_mf = nullptr, std::array const *consVar_fc_mf = nullptr); template static void ComputeFirstOrderFluxes(amrex::Array4 const &consVar, array_t &x1FluxDiffusive, amrex::Box const &indexRange); @@ -890,9 +891,9 @@ template void HydroSystem::SyncDualEnergy(amrex: template template -void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_mf, - amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, const amrex::Real K_visc, - amrex::MultiFab *x1FSpds_mf) +void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_cc_mf, + amrex::MultiFab const &x1RightState_cc_mf, amrex::MultiFab const &primVar_mf, const amrex::Real K_visc, + amrex::MultiFab *x1FSpds_mf, std::array const *consVar_fc_mf) { // By convention, the interfaces are defined on the left edge of each @@ -901,20 +902,54 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu // Indexing note: There are (nx + 1) interfaces for nx zones. - auto const &x1LeftState_in = x1LeftState_mf.const_arrays(); - auto const &x1RightState_in = x1RightState_mf.const_arrays(); + auto const &x1LeftState_cc_in = x1LeftState_cc_mf.const_arrays(); + auto const &x1RightState_cc_in = x1RightState_cc_mf.const_arrays(); auto const &primVar_in = primVar_mf.const_arrays(); auto x1Flux_in = x1Flux_mf.arrays(); auto x1FaceVel_in = x1FaceVel_mf.arrays(); + int delta_L_i = 0; + int delta_L_j = 0; + int delta_L_k = 0; + int delta_x2p_i = 0; + int delta_x2p_j = 0; + int delta_x2p_k = 0; + int delta_x3p_i = 0; + int delta_x3p_j = 0; + int delta_x3p_k = 0; amrex::MultiArray4 x1FSpds_in; + amrex::MultiArray4 x1State_fc_in; + amrex::MultiArray4 x2State_fc_in; + amrex::MultiArray4 x3State_fc_in; if constexpr (RIEMANN == RiemannSolver::HLLD) { x1FSpds_in = (*x1FSpds_mf).arrays(); + if constexpr (DIR == FluxDir::X1) { + delta_L_i = -1; + delta_x2p_j = 1; + delta_x3p_k = 1; + x1State_fc_in = (*consVar_fc_mf)[0].const_arrays(); + x2State_fc_in = (*consVar_fc_mf)[1].const_arrays(); + x3State_fc_in = (*consVar_fc_mf)[2].const_arrays(); + } else if constexpr (DIR == FluxDir::X2) { + delta_L_j = -1; + delta_x2p_k = 1; + delta_x3p_i = 1; + x1State_fc_in = (*consVar_fc_mf)[1].const_arrays(); + x2State_fc_in = (*consVar_fc_mf)[2].const_arrays(); + x3State_fc_in = (*consVar_fc_mf)[0].const_arrays(); + } else if constexpr (DIR == FluxDir::X3) { + delta_L_k = -1; + delta_x2p_i = 1; + delta_x3p_j = 1; + x1State_fc_in = (*consVar_fc_mf)[2].const_arrays(); + x2State_fc_in = (*consVar_fc_mf)[0].const_arrays(); + x3State_fc_in = (*consVar_fc_mf)[1].const_arrays(); + } } amrex::ParallelFor(x1Flux_mf, [=] AMREX_GPU_DEVICE(int bx, int i_in, int j_in, int k_in) { - quokka::Array4View x1LeftState(x1LeftState_in[bx]); - quokka::Array4View x1RightState(x1RightState_in[bx]); + quokka::Array4View x1LeftState(x1LeftState_cc_in[bx]); + quokka::Array4View x1RightState(x1RightState_cc_in[bx]); quokka::Array4View q(primVar_in[bx]); quokka::Array4View x1Flux(x1Flux_in[bx]); quokka::Array4View x1FaceVel(x1FaceVel_in[bx]); @@ -952,6 +987,12 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu double cs_L = NAN; double cs_R = NAN; + double bx1 = NAN; + double bx2_L = NAN; + double bx2_R = NAN; + double bx3_L = NAN; + double bx3_R = NAN; + if constexpr (is_eos_isothermal()) { P_L = rho_L * (cs_iso_ * cs_iso_); P_R = rho_R * (cs_iso_ * cs_iso_); @@ -991,6 +1032,46 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu E_R = quokka::EOS::ComputeEintFromPres(rho_R, P_R, massScalars_R) + ke_R; } + if constexpr (RIEMANN == RiemannSolver::HLLD) { + quokka::Array4View x1State_fc(x1State_fc_in[bx]); + quokka::Array4View x2State_fc(x2State_fc_in[bx]); + quokka::Array4View x3State_fc(x3State_fc_in[bx]); + + bx1 = x1State_fc(i_in, j_in, k_in, Physics_Indices::mhdFirstIndex); + + amrex::Print() << static_cast(DIR) << ": (" <<\ + i_in << "," << j_in << "," << k_in << "): " <<\ + bx1 << std::endl; + + const double bx2_Lm = x2State_fc(i_in+delta_L_i, j_in+delta_L_j, k_in+delta_L_k, Physics_Indices::mhdFirstIndex); + const double bx2_Lp = x2State_fc(i_in+delta_L_i+delta_x2p_i, j_in+delta_L_j+delta_x2p_j, k_in+delta_L_k+delta_x2p_k, Physics_Indices::mhdFirstIndex); + bx2_L = 0.5 * (bx2_Lm + bx2_Lp); + + amrex::Print() << static_cast(DIR) << ": (" <<\ + i_in << "," << j_in << "," << k_in << "): " <<\ + bx1 << std::endl; + + const double bx2_Rm = x2State_fc(i_in, j_in, k_in, Physics_Indices::mhdFirstIndex); + const double bx2_Rp = x2State_fc(i_in+delta_x2p_i, j_in+delta_x2p_j, k_in+delta_x2p_k, Physics_Indices::mhdFirstIndex); + bx2_R = 0.5 * (bx2_Rm + bx2_Rp); + + amrex::Print() << static_cast(DIR) << ": (" <<\ + i_in << "," << j_in << "," << k_in << "): " <<\ + bx1 << std::endl; + + const double bx3_Lm = x3State_fc(i_in+delta_L_i, j_in+delta_L_j, k_in+delta_L_k, Physics_Indices::mhdFirstIndex); + const double bx3_Lp = x3State_fc(i_in+delta_L_i+delta_x3p_i, j_in+delta_L_j+delta_x3p_j, k_in+delta_L_k+delta_x3p_k, Physics_Indices::mhdFirstIndex); + bx3_L = 0.5 * (bx3_Lm + bx3_Lp); + + amrex::Print() << static_cast(DIR) << ": (" <<\ + i_in << "," << j_in << "," << k_in << "): " <<\ + bx1 << std::endl; + + const double bx3_Rm = x3State_fc(i_in, j_in, k_in, Physics_Indices::mhdFirstIndex); + const double bx3_Rp = x3State_fc(i_in+delta_x3p_i, j_in+delta_x3p_j, k_in+delta_x3p_k, Physics_Indices::mhdFirstIndex); + bx3_R = 0.5 * (bx3_Rm + bx3_Rp); + } + AMREX_ASSERT(cs_L > 0.0); AMREX_ASSERT(cs_R > 0.0); @@ -1031,8 +1112,8 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu sL.Eint = Eint_L; // the following has been set to zero to test that the HLLD solver works with hydro only // TODO(Neco): set correct magnetic field values once magnetic fields are enabled - sL.by = 0.0; - sL.bz = 0.0; + sL.by = bx2_L; + sL.bz = bx3_L; quokka::HydroState sR{}; sR.rho = rho_R; @@ -1044,8 +1125,8 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu sR.E = E_R; sR.Eint = Eint_R; // as above, set to zero for testing purposes - sR.by = 0.0; - sR.bz = 0.0; + sR.by = bx2_R; + sR.bz = bx3_R; // The remaining components are mass scalars and passive scalars, so just copy them from // x1LeftState and x1RightState into the (left, right) state vectors U_L and @@ -1088,9 +1169,7 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu F_canonical = quokka::Riemann::LLF(sL, sR); } else if constexpr (RIEMANN == RiemannSolver::HLLD) { quokka::Array4View x1FSpds(x1FSpds_in[bx]); - // bx = 0 for testing purposes - // TODO(Neco): pass correct bx value once magnetic fields are enabled - auto [F_canonical, fspd_m, fspd_p] = quokka::Riemann::HLLD(sL, sR, gamma_, 0.0); + auto [F_canonical, fspd_m, fspd_p] = quokka::Riemann::HLLD(sL, sR, gamma_, bx1); x1FSpds(i, j, k, 0) = fspd_m; x1FSpds(i, j, k, 1) = fspd_p; } From 0227b20007d6486361c3fb13f70144bb295e41ad Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 5 Sep 2024 13:22:10 +1000 Subject: [PATCH 34/37] we have an extra cc-ghost for MHD problems --- src/simulation.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/simulation.hpp b/src/simulation.hpp index 54667b4ad..090251845 100644 --- a/src/simulation.hpp +++ b/src/simulation.hpp @@ -361,8 +361,7 @@ template class AMRSimulation : public amrex::AmrCore amrex::Vector>> fillpatcher_; // Nghost = number of ghost cells for each array - // TODO(Neco): 5 for MHD and 4 for hydro. so should cc = fc = 5 if MHD? - int nghost_cc_ = 4; // PPM needs nghost >= 3, PPM+flattening needs nghost >= 4 + int nghost_cc_ = Physics_Traits::is_mhd_enabled ? 5: 4; // PPM needs nghost >= 3, PPM+flattening needs nghost >= 4 int nghost_fc_ = Physics_Traits::is_mhd_enabled ? 4 : 2; // 4 needed for MHD, otherwise only 2 for tracer particles amrex::Vector componentNames_cc_; amrex::Vector componentNames_fc_; From 9b2509bea937403ca7571d6985aa6ba73a6b9138 Mon Sep 17 00:00:00 2001 From: neco kriel Date: Thu, 5 Sep 2024 13:25:16 +1000 Subject: [PATCH 35/37] being pedantic about syntax --- src/simulation.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation.hpp b/src/simulation.hpp index 090251845..ecdaf00c6 100644 --- a/src/simulation.hpp +++ b/src/simulation.hpp @@ -361,7 +361,7 @@ template class AMRSimulation : public amrex::AmrCore amrex::Vector>> fillpatcher_; // Nghost = number of ghost cells for each array - int nghost_cc_ = Physics_Traits::is_mhd_enabled ? 5: 4; // PPM needs nghost >= 3, PPM+flattening needs nghost >= 4 + int nghost_cc_ = Physics_Traits::is_mhd_enabled ? 5 : 4; // PPM needs nghost >= 3, PPM+flattening needs nghost >= 4 int nghost_fc_ = Physics_Traits::is_mhd_enabled ? 4 : 2; // 4 needed for MHD, otherwise only 2 for tracer particles amrex::Vector componentNames_cc_; amrex::Vector componentNames_fc_; From dd70e5641a39632eeab7c94698665d4e18f58feb Mon Sep 17 00:00:00 2001 From: neco kriel Date: Mon, 30 Sep 2024 16:09:23 +1000 Subject: [PATCH 36/37] update backend for mhd --- src/RadhydroSimulation.hpp | 79 +++++++++++++++++++++----- src/hydro_system.hpp | 113 ++++++++----------------------------- tests/alfven_wave.in | 2 +- 3 files changed, 90 insertions(+), 104 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 092c08857..bd2cfcd8d 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -270,7 +270,7 @@ template class RadhydroSimulation : public AMRSimulation dx); template - void hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, + void hydroFluxFunction(amrex::MultiFab &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, amrex::MultiFab &x1FaceVel, amrex::MultiFab &x1FSpds, std::array const &consVar_fc, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, int ng_reconstruct, int nvars); @@ -1071,7 +1071,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o if constexpr (Physics_Traits::is_mhd_enabled) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { auto ba_fc = amrex::convert(ba_cc, amrex::IntVect::TheDimensionVector(idim)); - state_inter_fc_[idim].define(ba_fc, dm, n_mhd_vars_per_dim_, nghost_fc_); + state_inter_fc_[idim].define(ba_fc, dm, Physics_Indices::nvarPerDim_fc, nghost_fc_); state_inter_fc_[idim].setVal(0); } } @@ -1122,7 +1122,12 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto const &stateOld_fc = state_old_fc_tmp; auto &stateNew_fc = state_inter_fc_; - auto [fluxArrays, faceVel, fast_mhd_wavespeeds] = computeHydroFluxes(stateOld_cc, stateOld_fc, ncompHydro_, lev); + int nvars = ncompHydro_; + if constexpr (Physics_Traits::is_mhd_enabled) { + // also create space to append the two magnetic field components orthogonal to the solving direction + nvars += 3; // add-hoc +3 because +1 (primScalar0_index) precedes +2 (x2Magnetic_index and x3Magnetic_index) at the end of the enum of primitive quantities + } + auto [fluxArrays, faceVel, fast_mhd_wavespeeds] = computeHydroFluxes(stateOld_cc, stateOld_fc, nvars, lev); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); @@ -1251,7 +1256,12 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto &stateFinal_cc = state_new_cc_[lev]; - auto [fluxArrays, faceVel, fast_mhd_wavespeeds] = computeHydroFluxes(stateInter_cc, stateInter_fc, ncompHydro_, lev); + int nvars = ncompHydro_; + if constexpr (Physics_Traits::is_mhd_enabled) { + // also create space to append the two magnetic field components orthogonal to the solving direction + nvars += 3; // add-hoc step: +3 because primScalar0_index precedes x2Magnetic_index and x3Magnetic_index at the end of the enum of primitive quantities + } + auto [fluxArrays, faceVel, fast_mhd_wavespeeds] = computeHydroFluxes(stateInter_cc, stateInter_fc, nvars, lev); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); @@ -1449,7 +1459,7 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co const int reconstructGhost = 1; // allocate temporary MultiFabs - amrex::MultiFab primVar(ba_cc, dm, nvars, nghost_cc_); + amrex::MultiFab primVar(ba_cc, dm, nvars, nghost_cc_); std::array flatCoefs; std::array flux; std::array facevel; @@ -1529,28 +1539,71 @@ auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &co template template -void RadhydroSimulation::hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, +void RadhydroSimulation::hydroFluxFunction(amrex::MultiFab &primVar_mf, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &flux, amrex::MultiFab &faceVel, amrex::MultiFab &x1FSpds, std::array const &consVar_fc, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, const int ng_reconstruct, const int nvars) { - if (reconstructionOrder_ == 3) { - HyperbolicSystem::template ReconstructStatesPPM(primVar, leftState, rightState, ng_reconstruct, nvars); + amrex::MultiArray4 x1State_fc_in; + amrex::MultiArray4 x2State_fc_in; + amrex::MultiArray4 x3State_fc_in; + if constexpr (Physics_Traits::is_mhd_enabled) { + std::array delta_x2 = {0, 0, 0}; + std::array delta_x3 = {0, 0, 0}; + if constexpr (DIR == FluxDir::X1) { + delta_x2[1] = 1; + delta_x3[2] = 1; + x1State_fc_in = consVar_fc[0].const_arrays(); + x2State_fc_in = consVar_fc[1].const_arrays(); + x3State_fc_in = consVar_fc[2].const_arrays(); + } else if constexpr (DIR == FluxDir::X2) { + delta_x2[2] = 1; + delta_x3[0] = 1; + x1State_fc_in = consVar_fc[1].const_arrays(); + x2State_fc_in = consVar_fc[2].const_arrays(); + x3State_fc_in = consVar_fc[0].const_arrays(); + } else if constexpr (DIR == FluxDir::X3) { + delta_x2[0] = 1; + delta_x3[1] = 1; + x1State_fc_in = consVar_fc[2].const_arrays(); + x2State_fc_in = consVar_fc[0].const_arrays(); + x3State_fc_in = consVar_fc[1].const_arrays(); + } + auto primVar_in = primVar_mf.arrays(); + amrex::ParallelFor(primVar_mf, [=] AMREX_GPU_DEVICE(int bx, int i_in, int j_in, int k_in) { + quokka::Array4View x2State_fc(x2State_fc_in[bx]); + quokka::Array4View x3State_fc(x3State_fc_in[bx]); + quokka::Array4View primVar(primVar_in[bx]); + + auto [i, j, k] = quokka::reorderMultiIndex(i_in, j_in, k_in); + + const double bx2_m = x2State_fc(i, j, k, Physics_Indices::mhdFirstIndex); + const double bx2_p = x2State_fc(i+delta_x2[0], j+delta_x2[1], k+delta_x2[2], Physics_Indices::mhdFirstIndex); + primVar(i, j, k, HydroSystem::x2Magnetic_index) = 0.5 * (bx2_m + bx2_p); + + const double bx3_m = x3State_fc(i, j, k, Physics_Indices::mhdFirstIndex); + const double bx3_p = x3State_fc(i+delta_x3[0], j+delta_x3[1], k+delta_x3[2], Physics_Indices::mhdFirstIndex); + primVar(i, j, k, HydroSystem::x3Magnetic_index) = 0.5 * (bx3_m + bx3_p); + }); + } + + if (reconstructionOrder_ == 3) { + HyperbolicSystem::template ReconstructStatesPPM(primVar_mf, leftState, rightState, ng_reconstruct, nvars); } else if (reconstructionOrder_ == 2) { - HyperbolicSystem::template ReconstructStatesPLM(primVar, leftState, rightState, ng_reconstruct, nvars); + HyperbolicSystem::template ReconstructStatesPLM(primVar_mf, leftState, rightState, ng_reconstruct, nvars); } else if (reconstructionOrder_ == 1) { - HyperbolicSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); + HyperbolicSystem::template ReconstructStatesConstant(primVar_mf, leftState, rightState, ng_reconstruct, nvars); } else { amrex::Abort("Invalid reconstruction order specified!"); } // cell-centered kernel - HydroSystem::template FlattenShocks(primVar, x1Flat, x2Flat, x3Flat, leftState, rightState, ng_reconstruct, nvars); + HydroSystem::template FlattenShocks(primVar_mf, x1Flat, x2Flat, x3Flat, leftState, rightState, ng_reconstruct, nvars); // interface-centered kernel if constexpr (Physics_Traits::is_mhd_enabled) { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, &x1FSpds, &consVar_fc); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar_mf, artificialViscosityK_, &x1FSpds, &consVar_fc[static_cast(DIR)]); } else { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_, nullptr, nullptr); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar_mf, artificialViscosityK_, nullptr, nullptr); } } diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 3c4b5677b..118463e1a 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -71,7 +71,9 @@ template class HydroSystem : public HyperbolicSystem 0!) + primScalar0_index, // first passive scalar (only present if nscalars > 0!) + x2Magnetic_index, // when magnetic fields exist, then we also store cc-ave of the two orthogonal b-fields, so they can be reconstructed to the solving face + x3Magnetic_index, }; static void ConservedToPrimitive(amrex::MultiFab const &cons_mf, amrex::MultiFab &primVar_mf, int nghost); @@ -120,7 +122,7 @@ template class HydroSystem : public HyperbolicSystem static void ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_cc_mf, amrex::MultiFab const &x1RightState_cc_mf, amrex::MultiFab const &primVar_mf, amrex::Real K_visc, - amrex::MultiFab *x1FSpds_mf = nullptr, std::array const *consVar_fc_mf = nullptr); + amrex::MultiFab *x1FSpds_mf = nullptr, amrex::MultiFab const *x1ConsVar_fc_mf = nullptr); template static void ComputeFirstOrderFluxes(amrex::Array4 const &consVar, array_t &x1FluxDiffusive, amrex::Box const &indexRange); @@ -893,7 +895,7 @@ template template void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_cc_mf, amrex::MultiFab const &x1RightState_cc_mf, amrex::MultiFab const &primVar_mf, const amrex::Real K_visc, - amrex::MultiFab *x1FSpds_mf, std::array const *consVar_fc_mf) + amrex::MultiFab *x1FSpds_mf, amrex::MultiFab const *x1ConsVar_fc_mf) { // By convention, the interfaces are defined on the left edge of each @@ -908,43 +910,11 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu auto x1Flux_in = x1Flux_mf.arrays(); auto x1FaceVel_in = x1FaceVel_mf.arrays(); - int delta_L_i = 0; - int delta_L_j = 0; - int delta_L_k = 0; - int delta_x2p_i = 0; - int delta_x2p_j = 0; - int delta_x2p_k = 0; - int delta_x3p_i = 0; - int delta_x3p_j = 0; - int delta_x3p_k = 0; amrex::MultiArray4 x1FSpds_in; - amrex::MultiArray4 x1State_fc_in; - amrex::MultiArray4 x2State_fc_in; - amrex::MultiArray4 x3State_fc_in; + amrex::MultiArray4 x1ConsVar_fc_in; if constexpr (RIEMANN == RiemannSolver::HLLD) { x1FSpds_in = (*x1FSpds_mf).arrays(); - if constexpr (DIR == FluxDir::X1) { - delta_L_i = -1; - delta_x2p_j = 1; - delta_x3p_k = 1; - x1State_fc_in = (*consVar_fc_mf)[0].const_arrays(); - x2State_fc_in = (*consVar_fc_mf)[1].const_arrays(); - x3State_fc_in = (*consVar_fc_mf)[2].const_arrays(); - } else if constexpr (DIR == FluxDir::X2) { - delta_L_j = -1; - delta_x2p_k = 1; - delta_x3p_i = 1; - x1State_fc_in = (*consVar_fc_mf)[1].const_arrays(); - x2State_fc_in = (*consVar_fc_mf)[2].const_arrays(); - x3State_fc_in = (*consVar_fc_mf)[0].const_arrays(); - } else if constexpr (DIR == FluxDir::X3) { - delta_L_k = -1; - delta_x2p_i = 1; - delta_x3p_j = 1; - x1State_fc_in = (*consVar_fc_mf)[2].const_arrays(); - x2State_fc_in = (*consVar_fc_mf)[0].const_arrays(); - x3State_fc_in = (*consVar_fc_mf)[1].const_arrays(); - } + x1ConsVar_fc_in = (*x1ConsVar_fc_mf).const_arrays(); } amrex::ParallelFor(x1Flux_mf, [=] AMREX_GPU_DEVICE(int bx, int i_in, int j_in, int k_in) { @@ -987,12 +957,6 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu double cs_L = NAN; double cs_R = NAN; - double bx1 = NAN; - double bx2_L = NAN; - double bx2_R = NAN; - double bx3_L = NAN; - double bx3_R = NAN; - if constexpr (is_eos_isothermal()) { P_L = rho_L * (cs_iso_ * cs_iso_); P_R = rho_R * (cs_iso_ * cs_iso_); @@ -1032,46 +996,6 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu E_R = quokka::EOS::ComputeEintFromPres(rho_R, P_R, massScalars_R) + ke_R; } - if constexpr (RIEMANN == RiemannSolver::HLLD) { - quokka::Array4View x1State_fc(x1State_fc_in[bx]); - quokka::Array4View x2State_fc(x2State_fc_in[bx]); - quokka::Array4View x3State_fc(x3State_fc_in[bx]); - - bx1 = x1State_fc(i_in, j_in, k_in, Physics_Indices::mhdFirstIndex); - - amrex::Print() << static_cast(DIR) << ": (" <<\ - i_in << "," << j_in << "," << k_in << "): " <<\ - bx1 << std::endl; - - const double bx2_Lm = x2State_fc(i_in+delta_L_i, j_in+delta_L_j, k_in+delta_L_k, Physics_Indices::mhdFirstIndex); - const double bx2_Lp = x2State_fc(i_in+delta_L_i+delta_x2p_i, j_in+delta_L_j+delta_x2p_j, k_in+delta_L_k+delta_x2p_k, Physics_Indices::mhdFirstIndex); - bx2_L = 0.5 * (bx2_Lm + bx2_Lp); - - amrex::Print() << static_cast(DIR) << ": (" <<\ - i_in << "," << j_in << "," << k_in << "): " <<\ - bx1 << std::endl; - - const double bx2_Rm = x2State_fc(i_in, j_in, k_in, Physics_Indices::mhdFirstIndex); - const double bx2_Rp = x2State_fc(i_in+delta_x2p_i, j_in+delta_x2p_j, k_in+delta_x2p_k, Physics_Indices::mhdFirstIndex); - bx2_R = 0.5 * (bx2_Rm + bx2_Rp); - - amrex::Print() << static_cast(DIR) << ": (" <<\ - i_in << "," << j_in << "," << k_in << "): " <<\ - bx1 << std::endl; - - const double bx3_Lm = x3State_fc(i_in+delta_L_i, j_in+delta_L_j, k_in+delta_L_k, Physics_Indices::mhdFirstIndex); - const double bx3_Lp = x3State_fc(i_in+delta_L_i+delta_x3p_i, j_in+delta_L_j+delta_x3p_j, k_in+delta_L_k+delta_x3p_k, Physics_Indices::mhdFirstIndex); - bx3_L = 0.5 * (bx3_Lm + bx3_Lp); - - amrex::Print() << static_cast(DIR) << ": (" <<\ - i_in << "," << j_in << "," << k_in << "): " <<\ - bx1 << std::endl; - - const double bx3_Rm = x3State_fc(i_in, j_in, k_in, Physics_Indices::mhdFirstIndex); - const double bx3_Rp = x3State_fc(i_in+delta_x3p_i, j_in+delta_x3p_j, k_in+delta_x3p_k, Physics_Indices::mhdFirstIndex); - bx3_R = 0.5 * (bx3_Rm + bx3_Rp); - } - AMREX_ASSERT(cs_L > 0.0); AMREX_ASSERT(cs_R > 0.0); @@ -1110,10 +1034,13 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu sL.cs = cs_L; sL.E = E_L; sL.Eint = Eint_L; - // the following has been set to zero to test that the HLLD solver works with hydro only - // TODO(Neco): set correct magnetic field values once magnetic fields are enabled - sL.by = bx2_L; - sL.bz = bx3_L; + if constexpr (RIEMANN == RiemannSolver::HLLD) { + sL.by = x1LeftState(i, j, k, x2Magnetic_index); + sL.bz = x1LeftState(i, j, k, x3Magnetic_index); + } else { + sL.by = 0.0; + sL.bz = 0.0; + } quokka::HydroState sR{}; sR.rho = rho_R; @@ -1124,9 +1051,13 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu sR.cs = cs_R; sR.E = E_R; sR.Eint = Eint_R; - // as above, set to zero for testing purposes - sR.by = bx2_R; - sR.bz = bx3_R; + if constexpr (RIEMANN == RiemannSolver::HLLD) { + sR.by = x1RightState(i, j, k, x2Magnetic_index); + sR.bz = x1RightState(i, j, k, x3Magnetic_index); + } else { + sR.by = 0.0; + sR.bz = 0.0; + } // The remaining components are mass scalars and passive scalars, so just copy them from // x1LeftState and x1RightState into the (left, right) state vectors U_L and @@ -1169,6 +1100,8 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu F_canonical = quokka::Riemann::LLF(sL, sR); } else if constexpr (RIEMANN == RiemannSolver::HLLD) { quokka::Array4View x1FSpds(x1FSpds_in[bx]); + quokka::Array4View x1ConsVar_fc(x1ConsVar_fc_in[bx]); + const double bx1 = x1ConsVar_fc(i, j, k, Physics_Indices::mhdFirstIndex); auto [F_canonical, fspd_m, fspd_p] = quokka::Riemann::HLLD(sL, sR, gamma_, bx1); x1FSpds(i, j, k, 0) = fspd_m; x1FSpds(i, j, k, 1) = fspd_p; diff --git a/tests/alfven_wave.in b/tests/alfven_wave.in index da4d0415c..0d12672f9 100644 --- a/tests/alfven_wave.in +++ b/tests/alfven_wave.in @@ -13,7 +13,7 @@ amr.v = 0 # verbosity in Amr # ***************************************************************** # Resolution and refinement # ***************************************************************** -amr.n_cell = 12 8 4 +amr.n_cell = 16 12 8 amr.max_level = 0 # number of levels = max_level + 1 amr.blocking_factor = 4 # grid size must be divisible by this From 9e7a72cbaa88c135ecc168f66a8dcd2962966c6b Mon Sep 17 00:00:00 2001 From: neco kriel Date: Mon, 30 Sep 2024 16:14:39 +1000 Subject: [PATCH 37/37] check alfven wave test passes --- src/AlfvenWave/test_alfven_wave.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/AlfvenWave/test_alfven_wave.cpp b/src/AlfvenWave/test_alfven_wave.cpp index 5c71922c0..6606085bc 100644 --- a/src/AlfvenWave/test_alfven_wave.cpp +++ b/src/AlfvenWave/test_alfven_wave.cpp @@ -177,9 +177,16 @@ auto problem_main() -> int } } - RadhydroSimulation sim_write(BCs_cc, BCs_fc); - sim_write.setInitialConditions(); - sim_write.evolve(); + RadhydroSimulation sim(BCs_cc, BCs_fc); + sim.setInitialConditions(); + sim.evolve(); + + // Compute test success condition + int status = 0; + const double error_tol = 0.002; + if (sim.errorNorm_ > error_tol) { + status = 1; + } - return 0; + return status; }