From d1e265666a593495513f4ad69ed05a01aaa20ea1 Mon Sep 17 00:00:00 2001 From: Jonah Maxwell Miller Date: Thu, 15 Jun 2023 15:45:01 -0600 Subject: [PATCH 1/5] add precommfillderived --- src/interface/state_descriptor.hpp | 8 ++++++++ src/interface/update.hpp | 11 +++++++++++ src/mesh/mesh.cpp | 10 ++++++++++ 3 files changed, 29 insertions(+) diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index 3c84b9991f3a..555f489417b0 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -325,6 +325,12 @@ class StateDescriptor { bool FlagsPresent(std::vector const &flags, bool matchAny = false); + void PreCommFillDerived(MeshBlockData *rc) const { + if (PreCommFillDerivedBlock != nullptr) PreCommFillDerivedBlock(rc); + } + void PreCommFillDerived(MeshData *rc) const { + if (PreCommFillDerivedMesh != nullptr) PreCommFillDerivedMesh(rc); + } void PreFillDerived(MeshBlockData *rc) const { if (PreFillDerivedBlock != nullptr) PreFillDerivedBlock(rc); } @@ -375,6 +381,8 @@ class StateDescriptor { std::vector> amr_criteria; + std::function *rc)> PreCommFillDerivedBlock = nullptr; + std::function *rc)> PreCommFillDerivedMesh = nullptr; std::function *rc)> PreFillDerivedBlock = nullptr; std::function *rc)> PreFillDerivedMesh = nullptr; std::function *rc)> PostFillDerivedBlock = nullptr; diff --git a/src/interface/update.hpp b/src/interface/update.hpp index 6a16f8b5b3ba..769d50273006 100644 --- a/src/interface/update.hpp +++ b/src/interface/update.hpp @@ -282,6 +282,17 @@ TaskStatus EstimateTimestep(T *rc) { return TaskStatus::complete; } +template +TaskStatus PreCommFillDerived(T *rc) { + Kokkos::Profiling::pushRegion("Task_PreCommFillDerived"); + auto pm = rc->GetParentPointer(); + for (const auto &pkg : pm->packages.AllPackages()) { + pkg.second->PreCommFillDerived(rc); + } + Kokkos::Profiling::popRegion(); + return TaskStatus::complete; +} + template TaskStatus FillDerived(T *rc) { Kokkos::Profiling::pushRegion("Task_FillDerived"); diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index bcfc15c1a4db..35cea0f2b36d 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -1071,6 +1071,16 @@ void Mesh::Initialize(bool init_problem, ParameterInput *pin, ApplicationInput * [](auto &sp_block) { sp_block->SetAllVariablesToInitialized(); }); } + // Pre comm fill derrived + for (int i = 0; i < nmb; ++i) { + auto &mbd = block_list[i]->meshblock_data.Get(); + Update::PreCommFillDerived(mbd.get()); + } + for (int i = 0; num_partitions; ++i) { + auto &md = mesh_data.GetOrAdd("base", i); + Update::PreCommFillDerived(md.get()); + } + // Build densely populated communication tags tag_map.clear(); for (int i = 0; i < num_partitions; i++) { From ba5e44fad1c078b312cf30765fe2ee3c7d6e2bfb Mon Sep 17 00:00:00 2001 From: Jonah Maxwell Miller Date: Thu, 15 Jun 2023 15:50:18 -0600 Subject: [PATCH 2/5] CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 565a407cdee7..ce34d9c352fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Date: 2023-05-26 ### Added (new features/APIs/variables/...) +- [[PR 889]](https://github.com/parthenon-hpc-lab/parthenon/pull/889) Add PreCommFillDerived - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing - [[PR 830]](https://github.com/parthenon-hpc-lab/parthenon/pull/830) Add particle output - [[PR 840]](https://github.com/parthenon-hpc-lab/parthenon/pull/840) Generalized integrators infrastructure in a backwards compatible way From d45e4a53fed09e19727e37eace0470fea17431ab Mon Sep 17 00:00:00 2001 From: Jonah Maxwell Miller Date: Thu, 15 Jun 2023 15:53:04 -0600 Subject: [PATCH 3/5] oops wrong place for changelog addition --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce34d9c352fb..62d583641f99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Current develop ### Added (new features/APIs/variables/...) +- [[PR 889]](https://github.com/parthenon-hpc-lab/parthenon/pull/889) Add PreCommFillDerived - [[PR 872]](https://github.com/parthenon-hpc-lab/parthenon/pull/872) Boundary communication for non-cell centered fields - [[PR 877]](https://github.com/parthenon-hpc-lab/parthenon/pull/877) Add flat sparse packs - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing @@ -20,7 +21,6 @@ Date: 2023-05-26 ### Added (new features/APIs/variables/...) -- [[PR 889]](https://github.com/parthenon-hpc-lab/parthenon/pull/889) Add PreCommFillDerived - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing - [[PR 830]](https://github.com/parthenon-hpc-lab/parthenon/pull/830) Add particle output - [[PR 840]](https://github.com/parthenon-hpc-lab/parthenon/pull/840) Generalized integrators infrastructure in a backwards compatible way From e0bbd13fda72f869db2c7261484e1abd6fc0857f Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Thu, 15 Jun 2023 16:07:50 -0600 Subject: [PATCH 4/5] Update src/mesh/mesh.cpp Co-authored-by: Patrick Mullen <33095288+pdmullen@users.noreply.github.com> --- src/mesh/mesh.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 35cea0f2b36d..ee7788cded07 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -1071,7 +1071,7 @@ void Mesh::Initialize(bool init_problem, ParameterInput *pin, ApplicationInput * [](auto &sp_block) { sp_block->SetAllVariablesToInitialized(); }); } - // Pre comm fill derrived + // Pre comm fill derived for (int i = 0; i < nmb; ++i) { auto &mbd = block_list[i]->meshblock_data.Get(); Update::PreCommFillDerived(mbd.get()); From 7d7a006766501dc68ad185da671e7bba0e58defb Mon Sep 17 00:00:00 2001 From: Jonah Maxwell Miller Date: Mon, 19 Jun 2023 09:35:28 -0600 Subject: [PATCH 5/5] what will win? Thoughtful coding or one little angle-ey boi --- src/mesh/mesh.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index ee7788cded07..00d16e591a2b 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -1076,7 +1076,7 @@ void Mesh::Initialize(bool init_problem, ParameterInput *pin, ApplicationInput * auto &mbd = block_list[i]->meshblock_data.Get(); Update::PreCommFillDerived(mbd.get()); } - for (int i = 0; num_partitions; ++i) { + for (int i = 0; i < num_partitions; ++i) { auto &md = mesh_data.GetOrAdd("base", i); Update::PreCommFillDerived(md.get()); }