Add nonlinear iterations for FSI (#47)

* Adding fixed number of nonlinear iterations for FSI just for nalu-wind

* Adding fixed number of nonlinear iterations for FSI just for nalu-wind

* Read nonlinear iterations from file

Also fix bug where AMR-Wind should not do anything after the first nonlinear iteration for call_advance_timestep

* increment timer example

* fix missing ;

* Style

* fix typo

---------

Co-authored-by: ashesh2512 <36968394+ashesh2512@users.noreply.github.com>
Co-authored-by: psakiev <psakiev@sandia.gov>

app/exawind/exawind.cpp

@@ -177,6 +177,10 @@ int main(int argc, char** argv)
         node["additional_picard_iterations"]
             ? node["additional_picard_iterations"].as<int>()
             : 0;
+    const int nonlinear_its = node["nonlinear_iterations"]
+                                  ? node["nonlinear_iterations"].as<int>()
+                                  : 1;
+
     for (int i = 0; i < num_nwsolvers; i++) {
         if (nalu_comms.at(i) != MPI_COMM_NULL)
             sim.register_solver<exawind::NaluWind>(
@@ -195,7 +199,7 @@ int main(int argc, char** argv)
     sim.echo("Initializing overset simulation");
     sim.initialize();
     sim.echo("Initialization successful");
-    sim.run_timesteps(additional_picard_its, num_timesteps);
+    sim.run_timesteps(additional_picard_its, nonlinear_its, num_timesteps);
     sim.delete_solvers();
 
     if (amr_comm != MPI_COMM_NULL) {

src/AMRWind.cpp

@@ -70,16 +70,24 @@ void AMRWind::prepare_solver_epilog()
     m_incflo.prepare_for_time_integration();
 }
 
-void AMRWind::pre_advance_stage1()
+void AMRWind::pre_advance_stage1(size_t inonlin)
 {
-    m_incflo.sim().time().new_timestep();
-    m_incflo.regrid_and_update();
-    m_incflo.pre_advance_stage1();
+    if (inonlin < 1) {
+        m_incflo.sim().time().new_timestep();
+        m_incflo.regrid_and_update();
+        m_incflo.pre_advance_stage1();
+    }
 }
 
-void AMRWind::pre_advance_stage2() { m_incflo.pre_advance_stage2(); }
+void AMRWind::pre_advance_stage2(size_t inonlin)
+{
+    if (inonlin < 1) m_incflo.pre_advance_stage2();
+}
 
-void AMRWind::advance_timestep() { m_incflo.do_advance(); }
+void AMRWind::advance_timestep(size_t inonlin)
+{
+    if (inonlin < 1) m_incflo.do_advance();
+}
 
 void AMRWind::post_advance() { m_incflo.post_advance_work(); }
 

src/AMRWind.h

@@ -37,9 +37,9 @@ class AMRWind : public ExawindSolver
     void init_epilog() override;
     void prepare_solver_prolog() override;
     void prepare_solver_epilog() override;
-    void pre_advance_stage1() override;
-    void pre_advance_stage2() override;
-    void advance_timestep() override;
+    void pre_advance_stage1(size_t inonlin) override;
+    void pre_advance_stage2(size_t inonlin) override;
+    void advance_timestep(size_t inonlin) override;
     void additional_picard_iterations(const int) override{};
     void post_advance() override;
     void pre_overset_conn_work() override;

src/ExawindSolver.h

@@ -19,25 +19,25 @@ class ExawindSolver
     void call_init_epilog() { init_epilog(); };
     void call_prepare_solver_prolog() { prepare_solver_prolog(); };
     void call_prepare_solver_epilog() { prepare_solver_epilog(); };
-    void call_pre_advance_stage1()
+    void call_pre_advance_stage1(size_t inonlin)
     {
         const std::string name = "Pre";
         m_timers.tick(name);
-        pre_advance_stage1();
+        pre_advance_stage1(inonlin);
         m_timers.tock(name);
     };
-    void call_pre_advance_stage2()
+    void call_pre_advance_stage2(size_t inonlin)
     {
         const std::string name = "Pre";
         m_timers.tick(name);
-        pre_advance_stage2();
+        pre_advance_stage2(inonlin);
         m_timers.tock(name);
     };
-    void call_advance_timestep()
+    void call_advance_timestep(size_t inonlin)
     {
         const std::string name = "Solve";
         m_timers.tick(name);
-        advance_timestep();
+        advance_timestep(inonlin);
         m_timers.tock(name);
     };
     void call_additional_picard_iterations(const int n)
@@ -108,9 +108,9 @@ class ExawindSolver
     virtual void init_epilog() = 0;
     virtual void prepare_solver_prolog() = 0;
     virtual void prepare_solver_epilog() = 0;
-    virtual void pre_advance_stage1() = 0;
-    virtual void pre_advance_stage2() = 0;
-    virtual void advance_timestep() = 0;
+    virtual void pre_advance_stage1(size_t inonlin) = 0;
+    virtual void pre_advance_stage2(size_t inonlin) = 0;
+    virtual void advance_timestep(size_t inonlin) = 0;
     virtual void additional_picard_iterations(const int) = 0;
     virtual void post_advance() = 0;
     virtual void pre_overset_conn_work() = 0;

src/NaluWind.cpp

@@ -81,20 +81,22 @@ void NaluWind::prepare_solver_epilog()
         realm->output_converged_results();
 }
 
-void NaluWind::pre_advance_stage1()
+void NaluWind::pre_advance_stage1(size_t inonlin)
 {
-    m_sim.timeIntegrator_->pre_realm_advance_stage1();
+    m_sim.timeIntegrator_->pre_realm_advance_stage1(inonlin);
 }
 
-void NaluWind::pre_advance_stage2()
+void NaluWind::pre_advance_stage2(size_t inonlin)
 {
-    m_sim.timeIntegrator_->pre_realm_advance_stage2();
+    m_sim.timeIntegrator_->pre_realm_advance_stage2(inonlin);
 }
 
-void NaluWind::advance_timestep()
+void NaluWind::advance_timestep(size_t inonlin)
 {
-    for (auto* realm : m_sim.timeIntegrator_->realmVec_)
+    for (auto* realm : m_sim.timeIntegrator_->realmVec_) {
         realm->advance_time_step();
+        realm->process_multi_physics_transfer();
+    }
 }
 
 void NaluWind::additional_picard_iterations(const int n)

src/NaluWind.h

@@ -50,9 +50,9 @@ class NaluWind : public ExawindSolver
     void init_epilog() override;
     void prepare_solver_prolog() override;
     void prepare_solver_epilog() override;
-    void pre_advance_stage1() override;
-    void pre_advance_stage2() override;
-    void advance_timestep() override;
+    void pre_advance_stage1(size_t inonlin) override;
+    void pre_advance_stage2(size_t inonlin) override;
+    void advance_timestep(size_t inonlin) override;
     void additional_picard_iterations(const int) override;
     void post_advance() override;
     void pre_overset_conn_work() override;

src/OversetSimulation.cpp

@@ -120,7 +120,8 @@ void OversetSimulation::exchange_solution(bool increment_time)
     for (auto& ss : m_solvers) ss->call_update_solution();
 }
 
-void OversetSimulation::run_timesteps(const int add_pic_its, const int nsteps)
+void OversetSimulation::run_timesteps(
+    const int add_pic_its, const int nonlinear_its, const int nsteps)
 {
 
     if (!m_initialized) {
@@ -138,15 +139,19 @@ void OversetSimulation::run_timesteps(const int add_pic_its, const int nsteps)
 
         m_timers_exa.tick("TimeStep");
 
-        for (auto& ss : m_solvers) ss->call_pre_advance_stage1();
+        for (size_t inonlin = 0; inonlin < nonlinear_its; inonlin++) {
 
-        if (do_connectivity(nt)) perform_overset_connectivity();
+            for (auto& ss : m_solvers) ss->call_pre_advance_stage1(inonlin);
 
-        for (auto& ss : m_solvers) ss->call_pre_advance_stage2();
+            if (do_connectivity(nt)) perform_overset_connectivity();
 
-        exchange_solution();
+            for (auto& ss : m_solvers) ss->call_pre_advance_stage2(inonlin);
 
-        for (auto& ss : m_solvers) ss->call_advance_timestep();
+            bool increment_timer = inonlin > 0 ? true : false;
+            exchange_solution(increment_timer);
+
+            for (auto& ss : m_solvers) ss->call_advance_timestep(inonlin);
+        }
 
         if (add_pic_its > 0) {
             exchange_solution(true);

src/OversetSimulation.h

@@ -79,7 +79,10 @@ class OversetSimulation
     void exchange_solution(const bool increment_time = false);
 
     //! Run prescribed number of timesteps
-    void run_timesteps(const int add_pic_its, const int nsteps = 1);
+    void run_timesteps(
+        const int add_pic_its,
+        const int nonlinear_its = 1,
+        const int nsteps = 1);
 
     //! Print something
     void echo(const std::string& out) { m_printer.echo(out); }