diff --git a/CHANGELOG.md b/CHANGELOG.md index 565a407cdee7..62d583641f99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Current develop ### Added (new features/APIs/variables/...) +- [[PR 889]](https://github.com/parthenon-hpc-lab/parthenon/pull/889) Add PreCommFillDerived - [[PR 872]](https://github.com/parthenon-hpc-lab/parthenon/pull/872) Boundary communication for non-cell centered fields - [[PR 877]](https://github.com/parthenon-hpc-lab/parthenon/pull/877) Add flat sparse packs - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index 3c84b9991f3a..555f489417b0 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -325,6 +325,12 @@ class StateDescriptor { bool FlagsPresent(std::vector const &flags, bool matchAny = false); + void PreCommFillDerived(MeshBlockData *rc) const { + if (PreCommFillDerivedBlock != nullptr) PreCommFillDerivedBlock(rc); + } + void PreCommFillDerived(MeshData *rc) const { + if (PreCommFillDerivedMesh != nullptr) PreCommFillDerivedMesh(rc); + } void PreFillDerived(MeshBlockData *rc) const { if (PreFillDerivedBlock != nullptr) PreFillDerivedBlock(rc); } @@ -375,6 +381,8 @@ class StateDescriptor { std::vector> amr_criteria; + std::function *rc)> PreCommFillDerivedBlock = nullptr; + std::function *rc)> PreCommFillDerivedMesh = nullptr; std::function *rc)> PreFillDerivedBlock = nullptr; std::function *rc)> PreFillDerivedMesh = nullptr; std::function *rc)> PostFillDerivedBlock = nullptr; diff --git a/src/interface/update.hpp b/src/interface/update.hpp index 6a16f8b5b3ba..769d50273006 100644 --- a/src/interface/update.hpp +++ b/src/interface/update.hpp @@ -282,6 +282,17 @@ TaskStatus EstimateTimestep(T *rc) { return TaskStatus::complete; } +template +TaskStatus PreCommFillDerived(T *rc) { + Kokkos::Profiling::pushRegion("Task_PreCommFillDerived"); + auto pm = rc->GetParentPointer(); + for (const auto &pkg : pm->packages.AllPackages()) { + pkg.second->PreCommFillDerived(rc); + } + Kokkos::Profiling::popRegion(); + return TaskStatus::complete; +} + template TaskStatus FillDerived(T *rc) { Kokkos::Profiling::pushRegion("Task_FillDerived"); diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index bcfc15c1a4db..00d16e591a2b 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -1071,6 +1071,16 @@ void Mesh::Initialize(bool init_problem, ParameterInput *pin, ApplicationInput * [](auto &sp_block) { sp_block->SetAllVariablesToInitialized(); }); } + // Pre comm fill derived + for (int i = 0; i < nmb; ++i) { + auto &mbd = block_list[i]->meshblock_data.Get(); + Update::PreCommFillDerived(mbd.get()); + } + for (int i = 0; i < num_partitions; ++i) { + auto &md = mesh_data.GetOrAdd("base", i); + Update::PreCommFillDerived(md.get()); + } + // Build densely populated communication tags tag_map.clear(); for (int i = 0; i < num_partitions; i++) {