From 20610d1b9551e7df0cd9fcc3b57fa9994750d060 Mon Sep 17 00:00:00 2001 From: psakievich Date: Tue, 25 Jul 2023 10:18:02 -0600 Subject: [PATCH] Allow in memory manipulation of nalu yaml files (#51) * Adjust nalu to construct with yaml * Add logfile naming option * Add test case that shows syntax * Style * Add comments and parallel check for file writing * Style * Fix logic error * Subtle fix for being careles with what I renamed in nalu --- .gitignore | 3 + app/exawind/exawind.cpp | 61 +++++- src/NaluWind.cpp | 14 +- src/NaluWind.h | 16 +- src/yaml-editor.h | 97 ++++++++++ test/CMakeLists.txt | 1 + .../hybrid-multi-cylinder/cylinder-amr.inp | 62 +++++++ .../hybrid-multi-cylinder/cylinder-nalu.yaml | 173 ++++++++++++++++++ .../hybrid-multi-cylinder.yaml | 55 ++++++ 9 files changed, 466 insertions(+), 16 deletions(-) create mode 100644 src/yaml-editor.h create mode 100644 test/test_files/hybrid-multi-cylinder/cylinder-amr.inp create mode 100644 test/test_files/hybrid-multi-cylinder/cylinder-nalu.yaml create mode 100644 test/test_files/hybrid-multi-cylinder/hybrid-multi-cylinder.yaml diff --git a/.gitignore b/.gitignore index e43b0f9..06b3fdb 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ .DS_Store +spack* +compile_commands.json +.cache diff --git a/app/exawind/exawind.cpp b/app/exawind/exawind.cpp index 4448ba0..5bfe588 100644 --- a/app/exawind/exawind.cpp +++ b/app/exawind/exawind.cpp @@ -3,6 +3,7 @@ #include "OversetSimulation.h" #include "MPIUtilities.h" #include "mpi.h" +#include "yaml-editor.h" #ifdef EXAWIND_HAS_STD_FILESYSTEM #include #endif @@ -88,8 +89,10 @@ int main(int argc, char** argv) #endif std::ofstream out; - const auto nalu_inps = node["nalu_wind_inp"].as>(); - const int num_nwsolvers = nalu_inps.size(); + YAML::Node nalu_node = node["nalu_wind_inp"]; + // make sure it is a list for now + assert(nalu_node.IsSequence()); + const int num_nwsolvers = nalu_node.size(); if (num_nwind_ranks < num_nwsolvers) { throw std::runtime_error( "Number of Nalu-Wind ranks is less than the number of Nalu-Wind " @@ -181,10 +184,60 @@ int main(int argc, char** argv) ? node["nonlinear_iterations"].as() : 1; + const YAML::Node yaml_replace_all = node["nalu_replace_all"]; for (int i = 0; i < num_nwsolvers; i++) { - if (nalu_comms.at(i) != MPI_COMM_NULL) + if (nalu_comms.at(i) != MPI_COMM_NULL) { + YAML::Node yaml_replace_instance; + YAML::Node this_instance = nalu_node[i]; + + std::string nalu_inpfile, logfile; + bool write_final_yaml_to_disk = false; + if (this_instance.IsMap()) { + yaml_replace_instance = this_instance["replace"]; + nalu_inpfile = + this_instance["base_input_file"].as(); + // deal with the logfile name + if (this_instance["logfile"]) { + logfile = this_instance["logfile"].as(); + } else { + logfile = exawind::NaluWind::change_file_name_suffix( + nalu_inpfile, ".log", i); + } + if (this_instance["write_final_yaml_to_disk"]) { + write_final_yaml_to_disk = + this_instance["write_final_yaml_to_disk"].as(); + } + + } else { + nalu_inpfile = this_instance.as(); + logfile = exawind::NaluWind::change_file_name_suffix( + nalu_inpfile, ".log"); + } + + YAML::Node nalu_yaml = YAML::LoadFile(nalu_inpfile); + // replace in order so instance can overwrite all + if (yaml_replace_all) { + YEDIT::find_and_replace(nalu_yaml, yaml_replace_all); + } + if (yaml_replace_instance) { + YEDIT::find_and_replace(nalu_yaml, yaml_replace_instance); + } + + // only the first rank of the comm should write the file + int comm_rank = -1; + MPI_Comm_rank(nalu_comms.at(i), &comm_rank); + if (write_final_yaml_to_disk && comm_rank == 0) { + auto new_ifile_name = + exawind::NaluWind::change_file_name_suffix( + logfile, ".yaml"); + std::ofstream fout(new_ifile_name); + fout << nalu_yaml; + fout.close(); + } + sim.register_solver( - i + 1, nalu_comms.at(i), nalu_inps.at(i), nalu_vars); + i + 1, nalu_comms.at(i), nalu_yaml, logfile, nalu_vars); + } } if (amr_comm != MPI_COMM_NULL) { diff --git a/src/NaluWind.cpp b/src/NaluWind.cpp index 723e53b..260f70e 100644 --- a/src/NaluWind.cpp +++ b/src/NaluWind.cpp @@ -32,14 +32,11 @@ void NaluWind::finalize() NaluWind::NaluWind( int id, stk::ParallelMachine comm, - const std::string& inpfile, + const YAML::Node& inp_yaml, + const std::string& logfile, const std::vector& fnames, TIOGA::tioga& tg) - : m_id(id) - , m_comm(comm) - , m_doc(YAML::LoadFile(inpfile)) - , m_fnames(fnames) - , m_sim(m_doc) + : m_id(id), m_comm(comm), m_doc(inp_yaml), m_fnames(fnames), m_sim(m_doc) { auto& env = sierra::nalu::NaluEnv::self(); env.parallelCommunicator_ = comm; @@ -48,11 +45,6 @@ NaluWind::NaluWind( ::tioga_nalu::TiogaRef::self(&tg); - int extloc = inpfile.rfind("."); - std::string logfile = inpfile; - if (extloc != std::string::npos) { - logfile = inpfile.substr(0, extloc) + ".log"; - } env.set_log_file_stream(logfile); } diff --git a/src/NaluWind.h b/src/NaluWind.h index b58739b..3bbde88 100644 --- a/src/NaluWind.h +++ b/src/NaluWind.h @@ -27,10 +27,24 @@ class NaluWind : public ExawindSolver public: static void initialize(); static void finalize(); + static std::string change_file_name_suffix( + std::string inpfile, std::string suffix, int index = -1) + { + int extloc = inpfile.rfind("."); + std::string logfile = inpfile; + if (index >= 0) { + suffix = "_" + std::to_string(index) + suffix; + } + if (extloc != std::string::npos) { + logfile = inpfile.substr(0, extloc) + suffix; + } + return logfile; + } explicit NaluWind( int id, stk::ParallelMachine comm, - const std::string& inp_file, + const YAML::Node& inp_yaml, + const std::string& logfile, const std::vector& fnames, TIOGA::tioga& tg); ~NaluWind(); diff --git a/src/yaml-editor.h b/src/yaml-editor.h new file mode 100644 index 0000000..47f1d44 --- /dev/null +++ b/src/yaml-editor.h @@ -0,0 +1,97 @@ +#include "yaml-cpp/yaml.h" +#include +#include + +namespace YEDIT { + +/* Exception if the graph is not formatted correctly + * This exception is designed to reproduce the failing portion of the YAML graph + * when called recursively so it can be echo'd to the user + */ +class YamlNodeMatchException : public std::exception +{ +public: + YamlNodeMatchException( + std::string currentNode, std::string accumulatedPath = "") + : std::exception() + { + std::string extra = + accumulatedPath.empty() ? "" : ":" + accumulatedPath; + graphPath_ = currentNode + extra; + } + const char* what() const noexcept override { return graphPath_.c_str(); } + +private: + std::string graphPath_; +}; + +namespace { +/* A function that will traverse the src node based on the key node + * The two nodes must be identical until the final leaf values and then the + * values of the key will override the src values. If the two graphs don't match + * at any point prior to the leaves then it will trigger a + * YamlNodeMatchException. + */ +inline void impl_find_and_replace(YAML::Node src, YAML::Node key) +{ + switch (key.Type()) { + case YAML::NodeType::Map: + // case 1: it's a map + //- pass the contents of the map recursively + for (auto n : key) { + std::string k = static_cast(n.first.Scalar()); + try { + impl_find_and_replace(src[k], key[k]); + } catch (YamlNodeMatchException& e) { + throw YamlNodeMatchException(k, std::string(e.what())); + } + } + break; + case YAML::NodeType::Sequence: + // case 2: it's a list + //- pass the contents of the list recursively (order matters when + // looking for a match) + for (int i = 0; i < key.size(); ++i) { + try { + impl_find_and_replace(src[i], key[i]); + } catch (YamlNodeMatchException& e) { + throw YamlNodeMatchException( + "[" + std::to_string(i) + "]", std::string(e.what())); + } + } + break; + case YAML::NodeType::Scalar: { + // case 3: it's a scalar + //- replace value + if (src.Type() != key.Type()) { + // we have a invalid path in the key + throw YamlNodeMatchException(key.as()); + } + src = key; + break; + } + default: + break; + } +} +} // namespace + +/* The accessible version of impl_find_and_replace that collects the final + * error and makes it more readable + */ +inline void find_and_replace(YAML::Node src, YAML::Node key) +{ + try { + impl_find_and_replace(src, key); + } catch (YamlNodeMatchException& e) { + auto failingPath = static_cast(e.what()); + std::string message = + "\nFailure trying to replace YAML\nFailing Graph:\n\t"; + message += failingPath + "\n"; + message += + "Please double check and make sure this matches the source YAML"; + throw std::runtime_error(message); + } +} + +} // namespace YEDIT diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fa68c7f..abdd174 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -61,3 +61,4 @@ add_test_r(nalu-nalu-cylinder) add_test_r(amr-nalu-cylinder) add_test_r(nalu-nalu-cylinder-motion) add_test_r(amr-nalu-cylinder-motion) +add_test_r(hybrid-multi-cylinder) diff --git a/test/test_files/hybrid-multi-cylinder/cylinder-amr.inp b/test/test_files/hybrid-multi-cylinder/cylinder-amr.inp new file mode 100644 index 0000000..112f68c --- /dev/null +++ b/test/test_files/hybrid-multi-cylinder/cylinder-amr.inp @@ -0,0 +1,62 @@ +# +# SIMULATION STOP # +#.......................................# +time.stop_time = 22000.0 # Max (# +tim.max_step = -1 +time.fixed_dt = 0.15 +time.cfl = 1.0 +time.plot_interval = 10 +time.checkpoint_interval = -10 +io.plot_file = out/plt +io.check_file = out/chk +# PHYSICS # +#.......................................# +incflo.gravity = 0. 0. 0.0 # Gravitational force (3D) + +incflo.use_godunov = 1 +incflo.do_initial_proj = 0 +incflo.initial_iterations = 0 +transport.viscosity = 0.005 +turbulence.model = Laminar + + +incflo.physics = FreeStream +incflo.velocity = 1.0 0.0 0.0 +incflo.density = 1.0 + +amr.n_cell = 160 56 4 # Grid cells at coarsest AMRlevel +amr.max_level = 0 # Max AMR level in hierarchy +amr.blocking_factor_z = 4 +amr.blocking_factor_x = 8 +amr.blocking_factor_y = 8 +amr.max_grid_size_z = 4 +amr.max_grid_size_x = 16 +amr.max_grid_size_y = 16 + +geometry.prob_lo = -15 -5.0 -0.375 # Lo corner coordinates +geometry.prob_hi = 7.5 5.0 0.375 # Hi corner coordinates +geometry.is_periodic = 0 0 1 # Periodicity x y z (0/1) + + +# Boundary conditions +xlo.type = "mass_inflow" +xlo.velocity = 1.0 0.0 0.0 +xlo.density = 1.0 +xhi.type = "pressure_outflow" +xhi.pressure = 0.0 +ylo.type = "slip_wall" +yhi.type = "slip_wall" +#zlo.type = "slip_wall" +#zhi.type = "slip_wall" + +incflo.verbose = 0 # incflo_level +amrex.fpe_trap_invalid = 0 # Trap NaNs +amrex.throw_exception = 1 +amrex.signal_handling = 0 + +mac_proj.verbose = 0 +nodal_proj.verbose = 0 +nodal_proj.mg_rtol = 1.0e-6 +nodal_proj.mg_atol = 1.0e-10 +nodal_proj.num_pre_smooth = 10 +nodal_proj.num_post_smooth = 10 diff --git a/test/test_files/hybrid-multi-cylinder/cylinder-nalu.yaml b/test/test_files/hybrid-multi-cylinder/cylinder-nalu.yaml new file mode 100644 index 0000000..5f4bb89 --- /dev/null +++ b/test/test_files/hybrid-multi-cylinder/cylinder-nalu.yaml @@ -0,0 +1,173 @@ +# -*- mode: yaml -*- + +Simulations: + - name: sim1 + time_integrator: ti_1 + optimizer: opt1 + +linear_solvers: + + - name: solve_scalar + type: tpetra + method: gmres + preconditioner: mt_sgs + tolerance: 1e-5 + max_iterations: 200 + kspace: 50 + output_level: 0 + + - name: solve_cont + type: hypre + method: hypre_gmres + preconditioner: boomerAMG + tolerance: 1e-5 + max_iterations: 200 + kspace: 5 + output_level: 0 + +realms: + - name: cylinder + mesh: meshes/cylinder3d.g + use_edges: yes + automatic_decomposition_type: rcb + + equation_systems: + name: theEqSys + max_iterations: 1 + decoupled_overset_solve: yes + + solver_system_specification: + velocity: solve_scalar + pressure: solve_cont + + systems: + + - LowMachEOM: + name: myLowMach + max_iterations: 1 + convergence_tolerance: 1e-7 + + mesh_motion: + - name: mover + mesh_parts: [block_2] + motion: + - type: rotation + omega: 0.1 + axis: [0.0, 0.0, 1.0] + centroid: [0.0, 0.0, 0.0] + + post_processing: + - type: surface + physics: surface_force_and_moment + output_file_name: forces.dat + frequency: 1 + parameters: [0, 0] + target_name: + - wall + + initial_conditions: + + - constant: ic_1 + target_name: + - block_2 + value: + pressure: 0.0 + velocity: [1.0, 0.0, 0.0] + + material_properties: + target_name: + - block_2 + specifications: + - name: density + type: constant + value: 1.00 + + - name: viscosity + type: constant + value: 0.005 + + boundary_conditions: + + - wall_boundary_condition: bc_5 + target_name: wall + wall_user_data: + velocity: [0.0, 0.0, 0.0] + + - periodic_boundary_condition: bc_6 + target_name: [cyl_zlo, cyl_zhi] + periodic_user_data: + search_tolerance: 1.e-2 + + - overset_boundary_condition: bc_overset + overset_connectivity_type: tioga + overset_user_data: + mesh_tag_offset: 1 + tioga_options: + symmetry_direction: 3 + set_resolutions: yes + mesh_group: + - overset_name: interior + mesh_parts: [ block_2] + wall_parts: [ wall ] + ovset_parts: [ overset ] + + mesh_transformation: + - name: init_position + mesh_parts: [ block_2 ] + motion: + - type: translation + displacement: [ 0.0, 0.0, 0.0] + + solution_options: + name: myOptions + projected_timescale_type: momentum_diag_inv #### Use 1/diagA formulation + + options: + - hybrid_factor: + velocity: 1.0 + + - upw_factor: + velocity: 1.0 + + - alpha_upw: + velocity: 1.0 + + - limiter: + pressure: no + velocity: no + + - projected_nodal_gradient: + pressure: element + velocity: element + + - relaxation_factor: + velocity: 0.7 + pressure: 0.3 + turbulent_ke: 0.7 + specific_dissipation_rate: 0.7 + output: + output_data_base_name: out/move-cylinder-near.e + output_frequency: 10 + output_node_set: no + output_variables: + - velocity + - pressure + - dpdx + - mesh_displacement + - iblank + - iblank_cell + + +Time_Integrators: + - StandardTimeIntegrator: + name: ti_1 + start_time: 0 + termination_step_count: 10000 + time_step: 0.15 + time_stepping_type: fixed + time_step_count: 0 + second_order_accuracy: yes + nonlinear_iterations: 4 + + realms: + - cylinder diff --git a/test/test_files/hybrid-multi-cylinder/hybrid-multi-cylinder.yaml b/test/test_files/hybrid-multi-cylinder/hybrid-multi-cylinder.yaml new file mode 100644 index 0000000..298587e --- /dev/null +++ b/test/test_files/hybrid-multi-cylinder/hybrid-multi-cylinder.yaml @@ -0,0 +1,55 @@ +# Example input file + +exawind: + # we can change things globally across all instances by providing a trace through + # the graph to the end values we want to edit + nalu_replace_all: + realms: + - name: replaced + boundary_conditions: + # null operator so we can skip sequence entries we don't want to change + - ~ + - ~ + - overset_boundary_condition: changed_overset_name + Time_Integrators: [ StandardTimeIntegrator: { realms: [replaced]}] + + nalu_wind_inp: + ################ + # front cylinder + ################ + - base_input_file: cylinder-nalu.yaml + logfile: stationary_front_cylinder.log + # set this value if you want to write the final nalu instances to disk + write_final_yaml_to_disk: true + # same syntax as nalu_replace_all but for this specific instance + replace: + realms: + - mesh_motion: + - motion: + - omega: 0.0 + post_processing: [{output_file_name: front_forces.dat}] + output: {output_data_base_name: out/front-cylinder.e} + mesh_transformation: [ motion: [displacement: [-7.0, 0.0, 0.0]]] + ################ + # back cylinder + ################ + - base_input_file: cylinder-nalu.yaml + logfile: moving_back_cylinder.log + write_final_yaml_to_disk: true + replace: + realms: + - post_processing: [{output_file_name: back_forces.dat}] + output: {output_data_base_name: out/back-cylinder.e} + + amr_wind_inp: cylinder-amr.inp + num_timesteps: 10 + additional_picard_iterations: 2 + + # Variables for overset exchange + nalu_vars: + - velocity + - pressure + amr_cell_vars: + - velocity + amr_node_vars: + - p