Merge branch 'main' into zalesak_example

.github/workflows/ci.yml

@@ -35,13 +35,14 @@ jobs:
         with:
           submodules: true
       - name: Tests
-        working-directory: /spack-manager/environments/exawind
         run: |
           /bin/bash -c " \
             source ${SPACK_MANAGER}/start.sh && \
+            quick-create -n ci -s exawind@master && \
+            spack cd -e && \
             ln -s ${GITHUB_WORKSPACE} exawind && \
-            source ${SPACK_MANAGER}/start.sh && \
-            quick-develop -s exawind@master && \
+            spack develop exawind@master && \
+            spack manager external --latest --blacklist gmake exawind
             spack install && \
             spack cd -b exawind && \
             spack build-env exawind ctest -j $(nproc) --output-on-failure \

.gitignore

@@ -1 +1,4 @@
 .DS_Store
+spack*
+compile_commands.json
+.cache

CMakeLists.txt

@@ -3,7 +3,6 @@ cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR)
 project(exawind CXX C)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE INTERNAL "")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
-include(exawind-utils)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_EXTENSIONS OFF)
@@ -18,6 +17,7 @@ option(EXAWIND_ENABLE_UMPIRE "Enable Umpire GPU memory pools" OFF)
 if(EXAWIND_ENABLE_CUDA)
   enable_language(CUDA)
   find_package(CUDAToolkit REQUIRED)
+  set(EXAWIND_CUDA_ARCH "70" CACHE STRING "CUDA architecture (Use 'Auto' for automatic detection)")
 endif()
 
 if(EXAWIND_ENABLE_ROCM)
@@ -70,6 +70,7 @@ add_executable(${EXAWIND_EXE_NAME})
 add_subdirectory(app/exawind)
 
 if(EXAWIND_ENABLE_CUDA)
+  include(exawind-utils)
   set(ewtargets "${EXAWIND_LIB_NAME};${EXAWIND_EXE_NAME}")
   foreach(tgt IN LISTS ewtargets)
     set_cuda_build_properties(${tgt})

app/exawind/exawind.cpp

@@ -3,6 +3,7 @@
 #include "OversetSimulation.h"
 #include "MPIUtilities.h"
 #include "mpi.h"
+#include "yaml-editor.h"
 #ifdef EXAWIND_HAS_STD_FILESYSTEM
 #include <filesystem>
 #endif
@@ -88,8 +89,10 @@ int main(int argc, char** argv)
 #endif
     std::ofstream out;
 
-    const auto nalu_inps = node["nalu_wind_inp"].as<std::vector<std::string>>();
-    const int num_nwsolvers = nalu_inps.size();
+    YAML::Node nalu_node = node["nalu_wind_inp"];
+    // make sure it is a list for now
+    assert(nalu_node.IsSequence());
+    const int num_nwsolvers = nalu_node.size();
     if (num_nwind_ranks < num_nwsolvers) {
         throw std::runtime_error(
             "Number of Nalu-Wind ranks is less than the number of Nalu-Wind "
@@ -177,10 +180,64 @@ int main(int argc, char** argv)
         node["additional_picard_iterations"]
             ? node["additional_picard_iterations"].as<int>()
             : 0;
+    const int nonlinear_its = node["nonlinear_iterations"]
+                                  ? node["nonlinear_iterations"].as<int>()
+                                  : 1;
+
+    const YAML::Node yaml_replace_all = node["nalu_replace_all"];
     for (int i = 0; i < num_nwsolvers; i++) {
-        if (nalu_comms.at(i) != MPI_COMM_NULL)
+        if (nalu_comms.at(i) != MPI_COMM_NULL) {
+            YAML::Node yaml_replace_instance;
+            YAML::Node this_instance = nalu_node[i];
+
+            std::string nalu_inpfile, logfile;
+            bool write_final_yaml_to_disk = false;
+            if (this_instance.IsMap()) {
+                yaml_replace_instance = this_instance["replace"];
+                nalu_inpfile =
+                    this_instance["base_input_file"].as<std::string>();
+                // deal with the logfile name
+                if (this_instance["logfile"]) {
+                    logfile = this_instance["logfile"].as<std::string>();
+                } else {
+                    logfile = exawind::NaluWind::change_file_name_suffix(
+                        nalu_inpfile, ".log", i);
+                }
+                if (this_instance["write_final_yaml_to_disk"]) {
+                    write_final_yaml_to_disk =
+                        this_instance["write_final_yaml_to_disk"].as<bool>();
+                }
+
+            } else {
+                nalu_inpfile = this_instance.as<std::string>();
+                logfile = exawind::NaluWind::change_file_name_suffix(
+                    nalu_inpfile, ".log");
+            }
+
+            YAML::Node nalu_yaml = YAML::LoadFile(nalu_inpfile);
+            // replace in order so instance can overwrite all
+            if (yaml_replace_all) {
+                YEDIT::find_and_replace(nalu_yaml, yaml_replace_all);
+            }
+            if (yaml_replace_instance) {
+                YEDIT::find_and_replace(nalu_yaml, yaml_replace_instance);
+            }
+
+            // only the first rank of the comm should write the file
+            int comm_rank = -1;
+            MPI_Comm_rank(nalu_comms.at(i), &comm_rank);
+            if (write_final_yaml_to_disk && comm_rank == 0) {
+                auto new_ifile_name =
+                    exawind::NaluWind::change_file_name_suffix(
+                        logfile, ".yaml");
+                std::ofstream fout(new_ifile_name);
+                fout << nalu_yaml;
+                fout.close();
+            }
+
             sim.register_solver<exawind::NaluWind>(
-                i + 1, nalu_comms.at(i), nalu_inps.at(i), nalu_vars);
+                i + 1, nalu_comms.at(i), nalu_yaml, logfile, nalu_vars);
+        }
     }
 
     if (amr_comm != MPI_COMM_NULL) {
@@ -195,7 +252,7 @@ int main(int argc, char** argv)
     sim.echo("Initializing overset simulation");
     sim.initialize();
     sim.echo("Initialization successful");
-    sim.run_timesteps(additional_picard_its, num_timesteps);
+    sim.run_timesteps(additional_picard_its, nonlinear_its, num_timesteps);
     sim.delete_solvers();
 
     if (amr_comm != MPI_COMM_NULL) {

cmake/exawind-utils.cmake

@@ -3,7 +3,7 @@ function(set_cuda_build_properties target)
   list(FILTER _tgt_src INCLUDE REGEX "\\.cpp")
   set_source_files_properties(${_tgt_src} PROPERTIES LANGUAGE CUDA)
   set_target_properties(${target} PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
-  set_target_properties(${target} PROPERTIES CUDA_ARCHITECTURES "70")
+  set_target_properties(${target} PROPERTIES CUDA_ARCHITECTURES "${EXAWIND_CUDA_ARCH}")
   set_target_properties(${target} PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
   #set_target_properties(${target} PROPERTIES LINKER_LANGUAGE CUDA)
 endfunction(set_cuda_build_properties)

src/AMRWind.cpp

@@ -70,16 +70,24 @@ void AMRWind::prepare_solver_epilog()
     m_incflo.prepare_for_time_integration();
 }
 
-void AMRWind::pre_advance_stage1()
+void AMRWind::pre_advance_stage1(size_t inonlin)
 {
-    m_incflo.sim().time().new_timestep();
-    m_incflo.regrid_and_update();
-    m_incflo.pre_advance_stage1();
+    if (inonlin < 1) {
+        m_incflo.sim().time().new_timestep();
+        m_incflo.regrid_and_update();
+        m_incflo.pre_advance_stage1();
+    }
 }
 
-void AMRWind::pre_advance_stage2() { m_incflo.pre_advance_stage2(); }
+void AMRWind::pre_advance_stage2(size_t inonlin)
+{
+    if (inonlin < 1) m_incflo.pre_advance_stage2();
+}
 
-void AMRWind::advance_timestep() { m_incflo.do_advance(); }
+void AMRWind::advance_timestep(size_t inonlin)
+{
+    if (inonlin < 1) m_incflo.do_advance();
+}
 
 void AMRWind::post_advance() { m_incflo.post_advance_work(); }
 

src/AMRWind.h

@@ -37,9 +37,9 @@ class AMRWind : public ExawindSolver
     void init_epilog() override;
     void prepare_solver_prolog() override;
     void prepare_solver_epilog() override;
-    void pre_advance_stage1() override;
-    void pre_advance_stage2() override;
-    void advance_timestep() override;
+    void pre_advance_stage1(size_t inonlin) override;
+    void pre_advance_stage2(size_t inonlin) override;
+    void advance_timestep(size_t inonlin) override;
     void additional_picard_iterations(const int) override{};
     void post_advance() override;
     void pre_overset_conn_work() override;

src/ExawindSolver.h

@@ -19,25 +19,25 @@ class ExawindSolver
     void call_init_epilog() { init_epilog(); };
     void call_prepare_solver_prolog() { prepare_solver_prolog(); };
     void call_prepare_solver_epilog() { prepare_solver_epilog(); };
-    void call_pre_advance_stage1()
+    void call_pre_advance_stage1(size_t inonlin)
     {
         const std::string name = "Pre";
         m_timers.tick(name);
-        pre_advance_stage1();
+        pre_advance_stage1(inonlin);
         m_timers.tock(name);
     };
-    void call_pre_advance_stage2()
+    void call_pre_advance_stage2(size_t inonlin)
     {
         const std::string name = "Pre";
         m_timers.tick(name);
-        pre_advance_stage2();
+        pre_advance_stage2(inonlin);
         m_timers.tock(name);
     };
-    void call_advance_timestep()
+    void call_advance_timestep(size_t inonlin)
     {
         const std::string name = "Solve";
         m_timers.tick(name);
-        advance_timestep();
+        advance_timestep(inonlin);
         m_timers.tock(name);
     };
     void call_additional_picard_iterations(const int n)
@@ -108,9 +108,9 @@ class ExawindSolver
     virtual void init_epilog() = 0;
     virtual void prepare_solver_prolog() = 0;
     virtual void prepare_solver_epilog() = 0;
-    virtual void pre_advance_stage1() = 0;
-    virtual void pre_advance_stage2() = 0;
-    virtual void advance_timestep() = 0;
+    virtual void pre_advance_stage1(size_t inonlin) = 0;
+    virtual void pre_advance_stage2(size_t inonlin) = 0;
+    virtual void advance_timestep(size_t inonlin) = 0;
     virtual void additional_picard_iterations(const int) = 0;
     virtual void post_advance() = 0;
     virtual void pre_overset_conn_work() = 0;

src/NaluWind.cpp

@@ -32,14 +32,11 @@ void NaluWind::finalize()
 NaluWind::NaluWind(
     int id,
     stk::ParallelMachine comm,
-    const std::string& inpfile,
+    const YAML::Node& inp_yaml,
+    const std::string& logfile,
     const std::vector<std::string>& fnames,
     TIOGA::tioga& tg)
-    : m_id(id)
-    , m_comm(comm)
-    , m_doc(YAML::LoadFile(inpfile))
-    , m_fnames(fnames)
-    , m_sim(m_doc)
+    : m_id(id), m_comm(comm), m_doc(inp_yaml), m_fnames(fnames), m_sim(m_doc)
 {
     auto& env = sierra::nalu::NaluEnv::self();
     env.parallelCommunicator_ = comm;
@@ -48,18 +45,22 @@ NaluWind::NaluWind(
 
     ::tioga_nalu::TiogaRef::self(&tg);
 
-    int extloc = inpfile.rfind(".");
-    std::string logfile = inpfile;
-    if (extloc != std::string::npos) {
-        logfile = inpfile.substr(0, extloc) + ".log";
-    }
     env.set_log_file_stream(logfile);
 }
 
 NaluWind::~NaluWind() = default;
 
 void NaluWind::init_prolog(bool multi_solver_mode)
 {
+    // Dump the input yaml to the start of the logfile
+    // before the nalu banner
+    auto& env = sierra::nalu::NaluEnv::self();
+    env.naluOutputP0() << std::string(20, '#') << " INPUT FILE START "
+                       << std::string(20, '#') << std::endl;
+    sierra::nalu::NaluParsingHelper::emit(*env.naluLogStream_, m_doc);
+    env.naluOutputP0() << std::string(20, '#') << " INPUT FILE END   "
+                       << std::string(20, '#') << std::endl;
+
     m_sim.load(m_doc);
     if (m_sim.timeIntegrator_->overset_ != nullptr)
         m_sim.timeIntegrator_->overset_->set_multi_solver_mode(
@@ -81,20 +82,22 @@ void NaluWind::prepare_solver_epilog()
         realm->output_converged_results();
 }
 
-void NaluWind::pre_advance_stage1()
+void NaluWind::pre_advance_stage1(size_t inonlin)
 {
-    m_sim.timeIntegrator_->pre_realm_advance_stage1();
+    m_sim.timeIntegrator_->pre_realm_advance_stage1(inonlin);
 }
 
-void NaluWind::pre_advance_stage2()
+void NaluWind::pre_advance_stage2(size_t inonlin)
 {
-    m_sim.timeIntegrator_->pre_realm_advance_stage2();
+    m_sim.timeIntegrator_->pre_realm_advance_stage2(inonlin);
 }
 
-void NaluWind::advance_timestep()
+void NaluWind::advance_timestep(size_t inonlin)
 {
-    for (auto* realm : m_sim.timeIntegrator_->realmVec_)
+    for (auto* realm : m_sim.timeIntegrator_->realmVec_) {
         realm->advance_time_step();
+        realm->process_multi_physics_transfer();
+    }
 }
 
 void NaluWind::additional_picard_iterations(const int n)

src/NaluWind.h

@@ -27,10 +27,24 @@ class NaluWind : public ExawindSolver
 public:
     static void initialize();
     static void finalize();
+    static std::string change_file_name_suffix(
+        std::string inpfile, std::string suffix, int index = -1)
+    {
+        int extloc = inpfile.rfind(".");
+        std::string logfile = inpfile;
+        if (index >= 0) {
+            suffix = "_" + std::to_string(index) + suffix;
+        }
+        if (extloc != std::string::npos) {
+            logfile = inpfile.substr(0, extloc) + suffix;
+        }
+        return logfile;
+    }
     explicit NaluWind(
         int id,
         stk::ParallelMachine comm,
-        const std::string& inp_file,
+        const YAML::Node& inp_yaml,
+        const std::string& logfile,
         const std::vector<std::string>& fnames,
         TIOGA::tioga& tg);
     ~NaluWind();
@@ -50,9 +64,9 @@ class NaluWind : public ExawindSolver
     void init_epilog() override;
     void prepare_solver_prolog() override;
     void prepare_solver_epilog() override;
-    void pre_advance_stage1() override;
-    void pre_advance_stage2() override;
-    void advance_timestep() override;
+    void pre_advance_stage1(size_t inonlin) override;
+    void pre_advance_stage2(size_t inonlin) override;
+    void advance_timestep(size_t inonlin) override;
     void additional_picard_iterations(const int) override;
     void post_advance() override;
     void pre_overset_conn_work() override;

src/OversetSimulation.cpp

@@ -120,7 +120,8 @@ void OversetSimulation::exchange_solution(bool increment_time)
     for (auto& ss : m_solvers) ss->call_update_solution();
 }
 
-void OversetSimulation::run_timesteps(const int add_pic_its, const int nsteps)
+void OversetSimulation::run_timesteps(
+    const int add_pic_its, const int nonlinear_its, const int nsteps)
 {
 
     if (!m_initialized) {
@@ -138,15 +139,19 @@ void OversetSimulation::run_timesteps(const int add_pic_its, const int nsteps)
 
         m_timers_exa.tick("TimeStep");
 
-        for (auto& ss : m_solvers) ss->call_pre_advance_stage1();
+        for (size_t inonlin = 0; inonlin < nonlinear_its; inonlin++) {
 
-        if (do_connectivity(nt)) perform_overset_connectivity();
+            for (auto& ss : m_solvers) ss->call_pre_advance_stage1(inonlin);
 
-        for (auto& ss : m_solvers) ss->call_pre_advance_stage2();
+            if (do_connectivity(nt)) perform_overset_connectivity();
 
-        exchange_solution();
+            for (auto& ss : m_solvers) ss->call_pre_advance_stage2(inonlin);
 
-        for (auto& ss : m_solvers) ss->call_advance_timestep();
+            bool increment_timer = inonlin > 0 ? true : false;
+            exchange_solution(increment_timer);
+
+            for (auto& ss : m_solvers) ss->call_advance_timestep(inonlin);
+        }
 
         if (add_pic_its > 0) {
             exchange_solution(true);