add precommfillderived

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Current develop
 
 ### Added (new features/APIs/variables/...)
+- [[PR 889]](https://github.com/parthenon-hpc-lab/parthenon/pull/889) Add PreCommFillDerived
 - [[PR 872]](https://github.com/parthenon-hpc-lab/parthenon/pull/872) Boundary communication for non-cell centered fields
 - [[PR 877]](https://github.com/parthenon-hpc-lab/parthenon/pull/877) Add flat sparse packs
 - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing

src/interface/state_descriptor.hpp

@@ -325,6 +325,12 @@ class StateDescriptor {
 
   bool FlagsPresent(std::vector<MetadataFlag> const &flags, bool matchAny = false);
 
+  void PreCommFillDerived(MeshBlockData<Real> *rc) const {
+    if (PreCommFillDerivedBlock != nullptr) PreCommFillDerivedBlock(rc);
+  }
+  void PreCommFillDerived(MeshData<Real> *rc) const {
+    if (PreCommFillDerivedMesh != nullptr) PreCommFillDerivedMesh(rc);
+  }
   void PreFillDerived(MeshBlockData<Real> *rc) const {
     if (PreFillDerivedBlock != nullptr) PreFillDerivedBlock(rc);
   }
@@ -375,6 +381,8 @@ class StateDescriptor {
 
   std::vector<std::shared_ptr<AMRCriteria>> amr_criteria;
 
+  std::function<void(MeshBlockData<Real> *rc)> PreCommFillDerivedBlock = nullptr;
+  std::function<void(MeshData<Real> *rc)> PreCommFillDerivedMesh = nullptr;
   std::function<void(MeshBlockData<Real> *rc)> PreFillDerivedBlock = nullptr;
   std::function<void(MeshData<Real> *rc)> PreFillDerivedMesh = nullptr;
   std::function<void(MeshBlockData<Real> *rc)> PostFillDerivedBlock = nullptr;

src/interface/update.hpp

@@ -282,6 +282,17 @@ TaskStatus EstimateTimestep(T *rc) {
   return TaskStatus::complete;
 }
 
+template <typename T>
+TaskStatus PreCommFillDerived(T *rc) {
+  Kokkos::Profiling::pushRegion("Task_PreCommFillDerived");
+  auto pm = rc->GetParentPointer();
+  for (const auto &pkg : pm->packages.AllPackages()) {
+    pkg.second->PreCommFillDerived(rc);
+  }
+  Kokkos::Profiling::popRegion();
+  return TaskStatus::complete;
+}
+
 template <typename T>
 TaskStatus FillDerived(T *rc) {
   Kokkos::Profiling::pushRegion("Task_FillDerived");

src/mesh/mesh.cpp

@@ -1071,6 +1071,16 @@ void Mesh::Initialize(bool init_problem, ParameterInput *pin, ApplicationInput *
                     [](auto &sp_block) { sp_block->SetAllVariablesToInitialized(); });
     }
 
+    // Pre comm fill derrived
+    for (int i = 0; i < nmb; ++i) {
+      auto &mbd = block_list[i]->meshblock_data.Get();
+      Update::PreCommFillDerived(mbd.get());
+    }
+    for (int i = 0; num_partitions; ++i) {
+      auto &md = mesh_data.GetOrAdd("base", i);
+      Update::PreCommFillDerived(md.get());
+    }
+
     // Build densely populated communication tags
     tag_map.clear();
     for (int i = 0; i < num_partitions; i++) {