diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml new file mode 100644 index 000000000000..fb70653ccb5a --- /dev/null +++ b/.github/workflows/ci-macos.yml @@ -0,0 +1,46 @@ +name: CI macOS + +on: [pull_request] + +# Cancel "duplicated" workflows triggered by pushes to internal +# branches with associated PRs. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +env: + CTEST_OUTPUT_ON_FAILURE: 1 + CMAKE_BUILD_PARALLEL_LEVEL: 4 # num threads for build + +jobs: + build: + runs-on: macos-latest + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' # caching pip dependencies + + - name: Configure + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + + - name: Build + run: cmake --build build + + - name: Test + run: | + cd build + ctest -LE 'regression' + + - uses: actions/upload-artifact@v3 + with: + name: configure-log-unit-macos + path: build/CMakeFiles/CMakeOutput.log + retention-days: 3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c735f1d19ce..fa0c1d01db6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ ### Fixed (not changing behavior/API/variables/...) - [[PR 1079]](https://github.com/parthenon-hpc-lab/parthenon/pull/1079) Address XDMF/Visit Issues +- [[PR 1088]](https://github.com/parthenon-hpc-lab/parthenon/pull/1088) Correctly fill fluxes for non-cell variables in SparsePacks +- [[PR 1083]](https://github.com/parthenon-hpc-lab/parthenon/pull/1083) Correctly fill VariableFluxPack for edge fluxes in 2D +- [[PR 1087]](https://github.com/parthenon-hpc-lab/parthenon/pull/1087) Make sure InnerLoopPatternTVR is resolved on device properly when it is the default loop pattern - [[PR 1071]](https://github.com/parthenon-hpc-lab/parthenon/pull/1070) Fix bug in static mesh refinement related to redefinition of Mesh::root_level - [[PR 1073]](https://github.com/parthenon-hpc-lab/parthenon/pull/1073) Fix bug in AMR and sparse restarts - [[PR 1070]](https://github.com/parthenon-hpc-lab/parthenon/pull/1070) Correctly exclude flux vars from searches by default diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1f6b7b72a594..988a1ebc5a1f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -78,7 +78,7 @@ else() endif() if (${PAR_LOOP_INNER_LAYOUT} STREQUAL "TVR_INNER_LOOP") - set(PAR_LOOP_INNER_LAYOUT_TAG inner_loop_pattern_tvr_tag) + set(PAR_LOOP_INNER_LAYOUT_TAG "InnerLoopPatternTVR()") elseif (${PAR_LOOP_INNER_LAYOUT} STREQUAL "SIMDFOR_INNER_LOOP") set(PAR_LOOP_INNER_LAYOUT_TAG inner_loop_pattern_simdfor_tag) else() diff --git a/src/interface/sparse_pack.hpp b/src/interface/sparse_pack.hpp index f1c1fa1846c3..db6cb5782674 100644 --- a/src/interface/sparse_pack.hpp +++ b/src/interface/sparse_pack.hpp @@ -337,7 +337,7 @@ class SparsePack : public SparsePackBase { auto &flux(const int b, const int dir, const int idx) const { PARTHENON_DEBUG_REQUIRE(!flat_, "Accessor cannot be used for flat packs"); PARTHENON_DEBUG_REQUIRE(dir > 0 && dir < 4 && with_fluxes_, "Bad input to flux call"); - return pack_(dir, b, idx); + return pack_(dir - 1 + flx_idx_, b, idx); } KOKKOS_INLINE_FUNCTION @@ -345,14 +345,14 @@ class SparsePack : public SparsePackBase { const int i) const { PARTHENON_DEBUG_REQUIRE(!flat_, "Accessor cannot be used for flat packs"); PARTHENON_DEBUG_REQUIRE(dir > 0 && dir < 4 && with_fluxes_, "Bad input to flux call"); - return pack_(dir, b, idx)(k, j, i); + return pack_(dir - 1 + flx_idx_, b, idx)(k, j, i); } KOKKOS_INLINE_FUNCTION Real &flux(const int dir, const int idx, const int k, const int j, const int i) const { PARTHENON_DEBUG_REQUIRE(flat_, "Accessor must only be used for flat packs"); PARTHENON_DEBUG_REQUIRE(dir > 0 && dir < 4 && with_fluxes_, "Bad input to flux call"); - return pack_(dir, 0, idx)(k, j, i); + return pack_(dir - 1 + flx_idx_, 0, idx)(k, j, i); } KOKKOS_INLINE_FUNCTION @@ -362,7 +362,7 @@ class SparsePack : public SparsePackBase { PARTHENON_DEBUG_REQUIRE(!flat_, "Accessor cannot be used for flat packs"); PARTHENON_DEBUG_REQUIRE(dir > 0 && dir < 4 && with_fluxes_, "Bad input to flux call"); const int n = bounds_(0, b, idx.VariableIdx()) + idx.Offset(); - return pack_(dir, b, n)(k, j, i); + return pack_(dir - 1 + flx_idx_, b, n)(k, j, i); } template ::value)> @@ -371,7 +371,7 @@ class SparsePack : public SparsePackBase { PARTHENON_DEBUG_REQUIRE(!flat_, "Accessor cannot be used for flat packs"); PARTHENON_DEBUG_REQUIRE(dir > 0 && dir < 4 && with_fluxes_, "Bad input to flux call"); const int vidx = GetLowerBound(b, t) + t.idx; - return pack_(dir, b, vidx)(k, j, i); + return pack_(dir - 1 + flx_idx_, b, vidx)(k, j, i); } template diff --git a/src/interface/sparse_pack_base.cpp b/src/interface/sparse_pack_base.cpp index 50a85db68ea4..5c387973804e 100644 --- a/src/interface/sparse_pack_base.cpp +++ b/src/interface/sparse_pack_base.cpp @@ -110,6 +110,7 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc, int max_size = 0; int nblocks = 0; bool contains_face_or_edge = false; + bool contains_face_with_fluxes = false; int size = 0; // local var used to compute size/block ForEachBlock(pmd, include_block, [&](int b, mbd_t *pmbd) { if (!desc.flat) { @@ -122,8 +123,13 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc, if (uid_map.count(uid) > 0) { const auto pv = uid_map.at(uid); if (pv->IsAllocated()) { - if (pv->IsSet(Metadata::Face) || pv->IsSet(Metadata::Edge)) + if (pv->IsSet(Metadata::Edge)) contains_face_or_edge = true; + if (pv->IsSet(Metadata::Face)) { + if (pv->IsSet(Metadata::WithFluxes) && desc.with_fluxes) { + contains_face_with_fluxes = true; + } contains_face_or_edge = true; + } int prod = pv->GetDim(6) * pv->GetDim(5) * pv->GetDim(4); size += prod; // max size/block (or total size for flat) pack.size_ += prod; // total ragged size @@ -138,7 +144,11 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc, // Allocate the views int leading_dim = 1; - if (desc.with_fluxes) { + pack.flx_idx_ = 1; + if (contains_face_with_fluxes) { + leading_dim += 5; + pack.flx_idx_ = 3; + } else if (desc.with_fluxes) { leading_dim += 3; } else if (contains_face_or_edge) { leading_dim += 2; @@ -205,11 +215,11 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc, if (pv->IsSet(Metadata::Face)) { pack.pack_h_(0, b, idx).topological_element = - TopologicalElement::E1; + TopologicalElement::F1; pack.pack_h_(1, b, idx).topological_element = - TopologicalElement::E2; + TopologicalElement::F2; pack.pack_h_(2, b, idx).topological_element = - TopologicalElement::E3; + TopologicalElement::F3; } } else { // This is a cell, node, or a variable that doesn't have @@ -221,13 +231,16 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc, } if (pv->IsSet(Metadata::Vector)) pack.pack_h_(0, b, idx).vector_component = v + 1; + } - if (desc.with_fluxes && pv->IsSet(Metadata::WithFluxes)) { - pack.pack_h_(1, b, idx) = pvf->data.Get(0, t, u, v); - pack.pack_h_(2, b, idx) = pvf->data.Get(1, t, u, v); - pack.pack_h_(3, b, idx) = pvf->data.Get(2, t, u, v); + if (desc.with_fluxes && pv->IsSet(Metadata::WithFluxes)) { + pack.pack_h_(0 + pack.flx_idx_, b, idx) = pvf->data.Get(0, t, u, v); + if (!pv->IsSet(Metadata::Edge)) { + pack.pack_h_(1 + pack.flx_idx_, b, idx) = pvf->data.Get(1, t, u, v); + pack.pack_h_(2 + pack.flx_idx_, b, idx) = pvf->data.Get(2, t, u, v); } } + for (auto el : GetTopologicalElements(pack.pack_h_(0, b, idx).topological_type)) { pack.pack_h_(static_cast(el) % 3, b, idx).topological_element = diff --git a/src/interface/sparse_pack_base.hpp b/src/interface/sparse_pack_base.hpp index a47aca139085..53fc4e37d0b1 100644 --- a/src/interface/sparse_pack_base.hpp +++ b/src/interface/sparse_pack_base.hpp @@ -92,6 +92,7 @@ class SparsePackBase { bounds_h_t bounds_h_; coords_t coords_; + int flx_idx_; bool with_fluxes_; bool coarse_; bool flat_; diff --git a/src/interface/variable_pack.hpp b/src/interface/variable_pack.hpp index 3bd098c33ea4..fba75750f691 100644 --- a/src/interface/variable_pack.hpp +++ b/src/interface/variable_pack.hpp @@ -452,9 +452,14 @@ class VariableFluxPack : public VariablePack { // host only inline auto flux_alloc_status() const { return flux_alloc_status_; } - KOKKOS_FORCEINLINE_FUNCTION - const ViewOfParArrays &flux(const int dir) const { - assert(dir > 0 && dir <= this->ndim_); + template + KOKKOS_FORCEINLINE_FUNCTION const ViewOfParArrays &flux(const int dir) const { + assert(dir > 0); + if constexpr (TT == TopologicalType::Edge) { + assert(dir <= this->ndim_ || (this->ndim_ == 2 && dir == 3)); + } else { + assert(dir <= this->ndim_); + } return f_[dir - 1]; } @@ -471,10 +476,11 @@ class VariableFluxPack : public VariablePack { constexpr bool IsFluxAllocated(const int /*n*/) const { return true; } #endif - KOKKOS_FORCEINLINE_FUNCTION - T &flux(const int dir, const int n, const int k, const int j, const int i) const { + template + KOKKOS_FORCEINLINE_FUNCTION T &flux(const int dir, const int n, const int k, + const int j, const int i) const { assert(IsFluxAllocated(n)); - return flux(dir)(n)(k, j, i); + return flux(dir)(n)(k, j, i); } private: @@ -681,9 +687,15 @@ void FillFluxViews(const VariableVector &vars, const int ndim, for (int i = 0; i < v->GetDim(4); i++) { host_al(vindex) = v->IsAllocated(); if (v->IsAllocated()) { - host_f1(vindex) = v->data.Get(0, k, j, i); - if (ndim >= 2) host_f2(vindex) = v->data.Get(1, k, j, i); - if (ndim >= 3) host_f3(vindex) = v->data.Get(2, k, j, i); + if (v->IsSet(Metadata::Edge)) { + if (ndim >= 2) host_f3(vindex) = v->data.Get(2, k, j, i); + if (ndim >= 3) host_f2(vindex) = v->data.Get(1, k, j, i); + if (ndim >= 3) host_f1(vindex) = v->data.Get(0, k, j, i); + } else { + host_f1(vindex) = v->data.Get(0, k, j, i); + if (ndim >= 2) host_f2(vindex) = v->data.Get(1, k, j, i); + if (ndim >= 3) host_f3(vindex) = v->data.Get(2, k, j, i); + } } vindex++; diff --git a/src/utils/interpolation.hpp b/src/utils/interpolation.hpp index 4415ebcba563..994905749f46 100644 --- a/src/utils/interpolation.hpp +++ b/src/utils/interpolation.hpp @@ -6,18 +6,16 @@ // Interpolation copied/refactored from // https://github.com/lanl/phoebus and https://github.com/lanl/spiner //======================================================================================== -// © 2022. Triad National Security, LLC. All rights reserved. This -// program was produced under U.S. Government contract -// 89233218CNA000001 for Los Alamos National Laboratory (LANL), which -// is operated by Triad National Security, LLC for the U.S. -// Department of Energy/National Nuclear Security Administration. All -// rights in the program are reserved by Triad National Security, LLC, -// and the U.S. Department of Energy/National Nuclear Security -// Administration. The Government is granted for itself and others -// acting on its behalf a nonexclusive, paid-up, irrevocable worldwide -// license in this material to reproduce, prepare derivative works, -// distribute copies to the public, perform publicly and display -// publicly, and to permit others to do so. +// (C) (or copyright) 2024. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. #ifndef UTILS_INTERPOLATION_HPP_ #define UTILS_INTERPOLATION_HPP_ diff --git a/src/utils/robust.hpp b/src/utils/robust.hpp index fde3c28c37cf..1714d9359b17 100644 --- a/src/utils/robust.hpp +++ b/src/utils/robust.hpp @@ -5,18 +5,16 @@ //======================================================================================== // Copied from https://github.com/lanl/phoebus //======================================================================================== -// © 2022. Triad National Security, LLC. All rights reserved. This -// program was produced under U.S. Government contract -// 89233218CNA000001 for Los Alamos National Laboratory (LANL), which -// is operated by Triad National Security, LLC for the U.S. -// Department of Energy/National Nuclear Security Administration. All -// rights in the program are reserved by Triad National Security, LLC, -// and the U.S. Department of Energy/National Nuclear Security -// Administration. The Government is granted for itself and others -// acting on its behalf a nonexclusive, paid-up, irrevocable worldwide -// license in this material to reproduce, prepare derivative works, -// distribute copies to the public, perform publicly and display -// publicly, and to permit others to do so. +// (C) (or copyright) 2024. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. #ifndef UTILS_ROBUST_HPP_ #define UTILS_ROBUST_HPP_