Merge remote-tracking branch 'origin/develop' into pdmullen/xdmf-visit

CHANGELOG.md

@@ -4,7 +4,9 @@
 
 ### Added (new features/APIs/variables/...)
 - [[PR 1079]](https://github.com/parthenon-hpc-lab/parthenon/pull/1079) Address XDMF/Visit Issues
+- [[PR 1084]](https://github.com/parthenon-hpc-lab/parthenon/pull/1084) Properly free swarm boundary MPI requests
 - [[PR 1020]](https://github.com/parthenon-hpc-lab/parthenon/pull/1020) Add bi- and trilinear interpolation routines
+- [[PR 1081]](https://github.com/parthenon-hpc-lab/parthenon/pull/1081) Add GetSize and GetIndex to sparse pack
 - [[PR 1026]](https://github.com/parthenon-hpc-lab/parthenon/pull/1026) Particle BCs without relocatable device code
 - [[PR 1037]](https://github.com/parthenon-hpc-lab/parthenon/pull/1037) Add SwarmPacks
 - [[PR 1068]](https://github.com/parthenon-hpc-lab/parthenon/pull/1068) Add ability to dump sparse pack contents as a string

src/interface/sparse_pack.hpp

@@ -160,36 +160,54 @@ class SparsePack : public SparsePackBase {
   }
 
   // Host Bound overloads
-  KOKKOS_INLINE_FUNCTION int GetLowerBoundHost(const int b) const {
+  int GetLowerBoundHost(const int b) const {
     return (flat_ && (b > 0)) ? (bounds_h_(1, b - 1, nvar_) + 1) : 0;
   }
 
-  KOKKOS_INLINE_FUNCTION int GetUpperBoundHost(const int b) const {
-    return bounds_h_(1, b, nvar_);
-  }
+  int GetUpperBoundHost(const int b) const { return bounds_h_(1, b, nvar_); }
 
-  KOKKOS_INLINE_FUNCTION int GetLowerBoundHost(const int b, PackIdx idx) const {
+  int GetLowerBoundHost(const int b, PackIdx idx) const {
     static_assert(sizeof...(Ts) == 0);
     return bounds_h_(0, b, idx.VariableIdx());
   }
 
-  KOKKOS_INLINE_FUNCTION int GetUpperBoundHost(const int b, PackIdx idx) const {
+  int GetUpperBoundHost(const int b, PackIdx idx) const {
     static_assert(sizeof...(Ts) == 0);
     return bounds_h_(1, b, idx.VariableIdx());
   }
 
   template <class TIn, REQUIRES(IncludesType<TIn, Ts...>::value)>
-  KOKKOS_INLINE_FUNCTION int GetLowerBoundHost(const int b, const TIn &) const {
+  int GetLowerBoundHost(const int b, const TIn &) const {
     const int vidx = GetTypeIdx<TIn, Ts...>::value;
     return bounds_h_(0, b, vidx);
   }
 
   template <class TIn, REQUIRES(IncludesType<TIn, Ts...>::value)>
-  KOKKOS_INLINE_FUNCTION int GetUpperBoundHost(const int b, const TIn &) const {
+  int GetUpperBoundHost(const int b, const TIn &) const {
     const int vidx = GetTypeIdx<TIn, Ts...>::value;
     return bounds_h_(1, b, vidx);
   }
 
+  // Number of components of a variable on a block
+  template <typename T>
+  KOKKOS_INLINE_FUNCTION int GetSize(const int b, const T &t) const {
+    return GetUpperBound(b, t) - GetLowerBound(b, t) + 1;
+  }
+  template <typename T>
+  int GetSizeHost(const int b, const T &t) const {
+    return GetUpperBoundHost(b, t) - GetLowerBoundHost(b, t) + 1;
+  }
+
+  // Index in pack
+  template <typename TIn, REQUIRES(IncludesType<TIn, Ts...>::value)>
+  KOKKOS_INLINE_FUNCTION int GetIndex(const int b, const TIn &var) const {
+    return GetLowerBound(b, var) + var.idx;
+  }
+  template <typename TIn, REQUIRES(IncludesType<TIn, Ts...>::value)>
+  int GetIndexHost(const int b, const TIn &var) const {
+    return GetLowerBoundHost(b, var) + var.idx;
+  }
+
   /* Usage:
    * Contains(b, v1(), v2(), v3())
    *
@@ -213,19 +231,17 @@ class SparsePack : public SparsePackBase {
     return (... && Contains(b, Args()));
   }
   // Host versions
-  KOKKOS_INLINE_FUNCTION bool ContainsHost(const int b) const {
-    return GetUpperBoundHost(b) >= 0;
-  }
+  bool ContainsHost(const int b) const { return GetUpperBoundHost(b) >= 0; }
   template <typename T>
-  KOKKOS_INLINE_FUNCTION bool ContainsHost(const int b, const T t) const {
+  bool ContainsHost(const int b, const T t) const {
     return GetUpperBoundHost(b, t) >= 0;
   }
   template <typename... Args>
-  KOKKOS_INLINE_FUNCTION bool ContainsHost(const int b, Args... args) const {
+  bool ContainsHost(const int b, Args... args) const {
     return (... && ContainsHost(b, args));
   }
   template <typename... Args, REQUIRES(sizeof...(Args) > 0)>
-  KOKKOS_INLINE_FUNCTION bool ContainsHost(const int b) const {
+  bool ContainsHost(const int b) const {
     return (... && ContainsHost(b, Args()));
   }
 

src/interface/swarm_comms.cpp

@@ -644,7 +644,9 @@ void Swarm::ResetCommunication() {
 #ifdef MPI_PARALLEL
   for (int n = 0; n < pmb->neighbors.size(); n++) {
     NeighborBlock &nb = pmb->neighbors[n];
-    vbswarm->bd_var_.req_send[nb.bufid] = MPI_REQUEST_NULL;
+    if (vbswarm->bd_var_.req_send[nb.bufid] != MPI_REQUEST_NULL) {
+      MPI_Request_free(&(vbswarm->bd_var_.req_send[nb.bufid]));
+    }
   }
 #endif
 

src/utils/index_split.hpp

@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2023. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2023-2024. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -91,8 +91,12 @@ class IndexSplit {
   KOKKOS_INLINE_FUNCTION
   int get_max_ni() const { return ibe_entire_ + 1; }
   // TODO(@jdolence) these overestimate max size...should probably fix
+  KOKKOS_INLINE_FUNCTION
   int get_max_nj() const { return (jbe_entire_ + 1) / njp_ + 1; }
+  KOKKOS_INLINE_FUNCTION
   int get_max_nk() const { return (kbe_entire_ + 1) / nkp_ + 1; }
+  KOKKOS_INLINE_FUNCTION
+  int get_max_nij() const { return get_max_ni() * get_max_nj(); }
   // inner_size could be used to find the bounds for a loop that is collapsed over
   // 1, 2, or 3 dimensions by providing the right starting and stopping indices
   template <typename V>

tst/unit/test_sparse_pack.cpp

@@ -217,6 +217,9 @@ TEST_CASE("Test behavior of sparse packs", "[SparsePack]") {
         REQUIRE(pack.ContainsHost(2, v5()));
         REQUIRE(!pack.ContainsHost(2, v1(), v3(), v5()));
         REQUIRE(pack.ContainsHost<v1, v5>(2));
+        REQUIRE(pack.GetSizeHost(2, v1()) == 1);
+        REQUIRE(pack.GetSizeHost(2, v3()) == 0);
+        REQUIRE(pack.GetSizeHost(1, v3()) == 3);
       }
 
       THEN("A sparse pack correctly loads this data and can be read from v3 on all "