Remove pre and post balance mesh dumping, clean up some comments

docs/src/config/model.md

@@ -119,8 +119,8 @@ output folder. `"SaveStep"` of 1 specifies to save all iterations, while for exa
 
 `"MaximumImbalance" [1.0]` : The ratio between maximum number of elements on a processor and
 minimum number of elements on a processor before a rebalance is performed. A value of `2.0`
-would result in rebalancing occurring only if one processor had more than double the number
-of elements on another.
+would result in rebalancing occurring if one processor exceeds twice the number of elements
+on another processor.
 
 `"Nonconformal" [false]` : Whether the adaptation should use nonconformal refinement.
 `"Nonconformal"` is necessary to enable `"UseCoarsening"`.
@@ -138,5 +138,3 @@ enabled.
   - `"Partition" [""]`
   - `"ReorientTetMesh" [false]`
   - `"RemoveCurvature" [false]`
-  - `"WritePostBalanceMesh" [false]`
-  - `"WritePreBalanceMesh" [false]`

palace/drivers/basesolver.cpp

@@ -104,7 +104,7 @@ namespace
 {
 
 // Helper function that returns an array of indices corresponding to marked elements.
-mfem::Array<int> MarkedElements(double threshold, const mfem::Vector &e)
+mfem::Array<int> MarkedElements(double threshold, const Vector &e)
 {
   mfem::Array<int> ind;
   ind.Reserve(e.Size());
@@ -142,16 +142,6 @@ void RebalanceMesh(std::unique_ptr<mfem::ParMesh> &mesh, const IoData &iodata,
       iodata.NondimensionalizeMesh(*mesh);
     }
   }
-  if (param.write_pre_balance_mesh)
-  {
-    std::ofstream pfile(
-        mfem::MakeParFilename(output_dir + "prebalance.", Mpi::Rank(comm), ".mesh", width));
-    mesh->SetPrintShared(false);  // Do not mark processor boundaries in the save
-    BlockTimer bt(Timer::IO);
-    iodata.DimensionalizeMesh(*mesh);
-    mesh->ParPrint(pfile);
-    iodata.NondimensionalizeMesh(*mesh);
-  }
 
   // If there is more than one processor, may perform rebalancing.
   if (Mpi::Size(comm) > 1)
@@ -170,7 +160,6 @@ void RebalanceMesh(std::unique_ptr<mfem::ParMesh> &mesh, const IoData &iodata,
       if (mesh->Nonconforming() && param.write_serial_mesh)
       {
         // Do not need to duplicate the mesh, as rebalancing will undo this.
-        BlockTimer bt_post(Timer::POSTPRO);
         mfem::Array<int> serial_partition(mesh->GetNE());
         serial_partition = 0;
         mesh->Rebalance(serial_partition);
@@ -182,7 +171,7 @@ void RebalanceMesh(std::unique_ptr<mfem::ParMesh> &mesh, const IoData &iodata,
           mesh->Mesh::Print(serial);
           iodata.NondimensionalizeMesh(*mesh);
         }
-        MPI_Barrier(comm);
+        Mpi::Barrier(comm);
       }
 
       Mpi::Print("Ratio {:.3f} exceeds maximum allowed value {}: Performing rebalancing.\n",
@@ -197,24 +186,13 @@ void RebalanceMesh(std::unique_ptr<mfem::ParMesh> &mesh, const IoData &iodata,
         // root processor and then redistributed.
         mesh::RebalanceConformalMesh(mesh, iodata, serial_mesh_filename);
       }
-      if (param.write_post_balance_mesh)
-      {
-        BlockTimer bt_io(Timer::IO);
-        std::ofstream pfile(mfem::MakeParFilename(output_dir + "postbalance.",
-                                                  Mpi::Rank(comm), ".mesh", width));
-        mesh->SetPrintShared(false);  // Do not mark processor boundaries in the save
-        iodata.DimensionalizeMesh(*mesh);
-        mesh->ParPrint(pfile);
-        iodata.NondimensionalizeMesh(*mesh);
-      }
     }
     else if (param.write_serial_mesh)
     {
       if (mesh->Nonconforming())
       {
         // Given no rebalancing will be done, need to handle the serial write more
         // carefully. For NC, this requires creating a duplicate mesh.
-        BlockTimer bt_post(Timer::POSTPRO);
         mfem::ParMesh smesh(*mesh);
         mfem::Array<int> serial_partition(mesh->GetNE());
         smesh.FinalizeTopology();
@@ -229,6 +207,7 @@ void RebalanceMesh(std::unique_ptr<mfem::ParMesh> &mesh, const IoData &iodata,
           iodata.DimensionalizeMesh(smesh);
           smesh.Mesh::Print(serial);
         }
+        Mpi::Barrier(comm);
       }
       else
       {
@@ -321,7 +300,7 @@ BaseSolver::SolveEstimateMarkRefine(std::vector<std::unique_ptr<mfem::ParMesh>>
 
   while (indicators.Norml2(comm) > param.tolerance && !exhausted_resources())
   {
-    BlockTimer bt_adapt(Timer::ADAPT);
+    BlockTimer bt_adapt(Timer::ADAPTATION);
     Mpi::Print("Adaptation iteration {}: Initial Error Indicator: {:.3e}, DOF: {}, DOF "
                "Limit: {}\n",
                ++iter, indicators.Norml2(comm), ntdof, param.dof_limit);
@@ -345,7 +324,7 @@ BaseSolver::SolveEstimateMarkRefine(std::vector<std::unique_ptr<mfem::ParMesh>>
       // Perform a Dörfler style marking looking for the largest number of derefinement
       // opportunities to represent a fraction of the derefinable error.
       const auto &derefinement_table = mesh.back()->pncmesh->GetDerefinementTable();
-      mfem::Vector coarse_error(derefinement_table.Size());
+      Vector coarse_error(derefinement_table.Size());
       mfem::Array<int> row;
       for (int i = 0; i < derefinement_table.Size(); i++)
       {

palace/utils/communication.hpp

@@ -48,6 +48,7 @@ inline MPI_Datatype DataType<unsigned short>()
 {
   return MPI_UNSIGNED_SHORT;
 }
+
 template <>
 inline MPI_Datatype DataType<signed int>()
 {

palace/utils/configfile.cpp

@@ -411,10 +411,6 @@ void RefinementData::SetUp(json &model)
     adaptation.nonconformal = adapt->value("Nonconformal", adaptation.nonconformal);
     adaptation.maximum_imbalance =
         adapt->value("MaximumImbalance", adaptation.maximum_imbalance);
-    adaptation.write_post_balance_mesh =
-        adapt->value("WritePostBalanceMesh", adaptation.write_post_balance_mesh);
-    adaptation.write_pre_balance_mesh =
-        adapt->value("WritePreBalanceMesh", adaptation.write_pre_balance_mesh);
     adaptation.write_serial_mesh =
         adapt->value("WriteSerialMesh", adaptation.write_serial_mesh);
 
@@ -432,18 +428,9 @@ void RefinementData::SetUp(json &model)
                 "\"MaximumImbalance\" must be greater than or equal to 1");
 
     // Cleanup
-    const auto fields = {"Tol",
-                         "MaxIts",
-                         "UpdateFraction",
-                         "UseCoarsening",
-                         "MaxNCLevels",
-                         "DOFLimit",
-                         "SaveStep",
-                         "Nonconformal",
-                         "MaximumImbalance",
-                         "WritePostBalanceMesh",
-                         "WritePreBalanceMesh",
-                         "WriteSerialMesh"};
+    const auto fields = {
+        "Tol",      "MaxIts",   "UpdateFraction", "UseCoarsening",    "MaxNCLevels",
+        "DOFLimit", "SaveStep", "Nonconformal",   "MaximumImbalance", "WriteSerialMesh"};
     for (const auto &f : fields)
     {
       adapt->erase(f);

palace/utils/configfile.hpp

@@ -148,18 +148,14 @@ struct AdaptiveRefinementData
   int dof_limit = 0;
   // Frequency with which to store the post processing results for a given adaptation, e.g.
   // save_step = 3 means save every third adaptation.
-  int save_step = 0;
+  int save_step = 1;
   // Whether or not to perform nonconformal adaptation.
   bool nonconformal = false;
   // Maximum allowable ratio of number of elements across processors before rebalancing is
   // performed.
   double maximum_imbalance = 1.0;
   // Whether to write a serialized mesh to file after mesh modification.
   bool write_serial_mesh = true;
-  // Whether to write the refined mesh to file in parallel, before rebalancing.
-  bool write_pre_balance_mesh = false;
-  // Whether to write the refined mesh to file in parallel, after rebalancing.
-  bool write_post_balance_mesh = false;
 };
 
 struct RefinementData

palace/utils/dorfler.cpp

@@ -10,7 +10,7 @@
 namespace palace::utils
 {
 
-double ComputeDorflerThreshold(double fraction, const mfem::Vector &e)
+double ComputeDorflerThreshold(double fraction, const Vector &e)
 {
   // Precompute the sort and partial sum to make evaluating a candidate partition fast.
   e.HostRead();  // Pull the data out to the host
@@ -72,9 +72,8 @@ double ComputeDorflerThreshold(double fraction, const mfem::Vector &e)
               "min: " << min_threshold << " max " << max_threshold);
   auto [elem_marked, error_marked] = marked(error_threshold);
 
-  // Keep track of the number of elements marked by the threshold bounds. If the
-  // top and bottom values are equal (or separated by only 1), there's no point further
-  // bisecting.
+  // Keep track of the number of elements marked by the threshold bounds. If the top and
+  // bottom values are equal (or separated by only 1), there's no point further bisecting.
   auto [max_elem_marked, max_error_marked] = marked(min_threshold);
   auto [min_elem_marked, min_error_marked] = marked(max_threshold);
   Mpi::GlobalSum(1, &min_elem_marked, comm);
@@ -149,8 +148,8 @@ double ComputeDorflerThreshold(double fraction, const mfem::Vector &e)
   elem_marked = max_elem_marked;
   error_marked = max_error_marked;
 
-  Mpi::Print("Threshold {:.3e} marked {} of {} and {:.2f}%\n",
-             error_threshold, elem_marked, total_elem, 100 * error_marked / total_error);
+  Mpi::Print("Threshold {:.3e} marked {} of {} and {:.2f}%\n", error_threshold, elem_marked,
+             total_elem, 100 * error_marked / total_error);
 
   MFEM_VERIFY(error_marked >= fraction * total_error,
               "Marked error: " << error_marked << " total error: " << total_error

palace/utils/dorfler.hpp

@@ -4,7 +4,7 @@
 #ifndef PALACE_UTILS_DORFLER_HPP
 #define PALACE_UTILS_DORFLER_HPP
 
-#include <mfem.hpp>
+#include "linalg/vector.hpp"
 
 namespace palace::utils
 {
@@ -15,7 +15,7 @@ namespace palace::utils
 // will mark the top fraction of the sum of the squared error.
 // Reference: Willy Dörfler. A convergent adaptive algorithm for Poisson’s equation. SIAM J.
 //            Numer. Anal. (1996).
-double ComputeDorflerThreshold(double fraction, const mfem::Vector &e);
+double ComputeDorflerThreshold(double fraction, const Vector &e);
 
 }  // namespace palace::utils
 

palace/utils/geodata.cpp

@@ -95,14 +95,11 @@ std::unique_ptr<mfem::ParMesh> ReadMesh(MPI_Comm comm, const IoData &iodata, boo
     BlockTimer bt(Timer::IO);
     if (Mpi::Root(comm) || !use_mesh_partitioner)
     {
-      // If using the mesh partitioner, only the root node needs to load the mesh.
       // Optionally reorder elements (and vertices) based on spatial location after loading
       // the serial mesh.
       smesh = LoadMesh(iodata.model.mesh, iodata.model.remove_curvature);
       if (reorder)
       {
-        // Optionally reorder elements (and vertices) based on spatial location after
-        // loading the serial mesh.
         smesh = LoadMesh(iodata.model.mesh, iodata.model.remove_curvature);
         if (reorder)
         {

palace/utils/timer.hpp

@@ -35,7 +35,7 @@ class Timer
     ESTIMATION,          // Estimation
     CONSTRUCTESTIMATOR,  // Construction of estimator
     SOLVEESTIMATOR,      // Evaluation of estimator
-    ADAPT,               // Adaptation
+    ADAPTATION,          // Adaptation
     REBALANCE,           // Rebalancing
     CONSTRUCTPROM,       // Adaptive frequency sweep
     SOLVEPROM,           // Adaptive frequency sweep

scripts/schema/config/model.json

@@ -36,8 +36,6 @@
             "MaxNCLevels": {"type": "integer", "inclusiveMinimum": 0},
             "Nonconformal": {"type": "boolean"},
             "MaximumImbalance": {"type": "number", "inclusiveMinimum": 1.0},
-            "WritePostBalanceMesh": {"type": "boolean"},
-            "WritePreBalanceMesh": {"type": "boolean"},
             "WriteSerialMesh": {"type": "boolean"}
           }
         },