Move from utils to fem, CwiseSqrt, and more debris

palace/drivers/basesolver.cpp

@@ -6,12 +6,12 @@
 #include <complex>
 #include <mfem.hpp>
 #include <nlohmann/json.hpp>
+#include "fem/errorindicators.hpp"
 #include "linalg/ksp.hpp"
 #include "models/domainpostoperator.hpp"
 #include "models/postoperator.hpp"
 #include "models/surfacepostoperator.hpp"
 #include "utils/communication.hpp"
-#include "utils/errorindicators.hpp"
 #include "utils/filesystem.hpp"
 #include "utils/iodata.hpp"
 #include "utils/timer.hpp"

palace/drivers/drivensolver.cpp

@@ -5,6 +5,7 @@
 
 #include <complex>
 #include <mfem.hpp>
+#include "fem/errorindicators.hpp"
 #include "linalg/errorestimator.hpp"
 #include "linalg/ksp.hpp"
 #include "linalg/operator.hpp"
@@ -16,7 +17,6 @@
 #include "models/surfacecurrentoperator.hpp"
 #include "models/waveportoperator.hpp"
 #include "utils/communication.hpp"
-#include "utils/errorindicators.hpp"
 #include "utils/iodata.hpp"
 #include "utils/prettyprint.hpp"
 #include "utils/timer.hpp"

palace/drivers/eigensolver.cpp

@@ -4,6 +4,7 @@
 #include "eigensolver.hpp"
 
 #include <mfem.hpp>
+#include "fem/errorindicators.hpp"
 #include "linalg/arpack.hpp"
 #include "linalg/divfree.hpp"
 #include "linalg/errorestimator.hpp"
@@ -15,7 +16,6 @@
 #include "models/postoperator.hpp"
 #include "models/spaceoperator.hpp"
 #include "utils/communication.hpp"
-#include "utils/errorindicators.hpp"
 #include "utils/iodata.hpp"
 #include "utils/timer.hpp"
 

palace/drivers/electrostaticsolver.cpp

@@ -4,14 +4,14 @@
 #include "electrostaticsolver.hpp"
 
 #include <mfem.hpp>
+#include "fem/errorindicators.hpp"
 #include "linalg/errorestimator.hpp"
 #include "linalg/ksp.hpp"
 #include "linalg/operator.hpp"
 #include "linalg/vector.hpp"
 #include "models/laplaceoperator.hpp"
 #include "models/postoperator.hpp"
 #include "utils/communication.hpp"
-#include "utils/errorindicators.hpp"
 #include "utils/iodata.hpp"
 #include "utils/timer.hpp"
 

palace/drivers/magnetostaticsolver.cpp

@@ -4,14 +4,14 @@
 #include "magnetostaticsolver.hpp"
 
 #include <mfem.hpp>
+#include "fem/errorindicators.hpp"
 #include "linalg/errorestimator.hpp"
 #include "linalg/ksp.hpp"
 #include "linalg/operator.hpp"
 #include "models/curlcurloperator.hpp"
 #include "models/postoperator.hpp"
 #include "models/surfacecurrentoperator.hpp"
 #include "utils/communication.hpp"
-#include "utils/errorindicators.hpp"
 #include "utils/iodata.hpp"
 #include "utils/timer.hpp"
 

palace/drivers/transientsolver.cpp

@@ -4,6 +4,7 @@
 #include "transientsolver.hpp"
 
 #include <mfem.hpp>
+#include "fem/errorindicators.hpp"
 #include "linalg/errorestimator.hpp"
 #include "linalg/vector.hpp"
 #include "models/lumpedportoperator.hpp"
@@ -12,7 +13,6 @@
 #include "models/surfacecurrentoperator.hpp"
 #include "models/timeoperator.hpp"
 #include "utils/communication.hpp"
-#include "utils/errorindicators.hpp"
 #include "utils/excitations.hpp"
 #include "utils/iodata.hpp"
 #include "utils/timer.hpp"

palace/fem/CMakeLists.txt

@@ -5,6 +5,7 @@
 # Add source files and subdirectories.
 #
 
-# target_sources(${TARGET_NAME}
-#   PRIVATE
-# )
+target_sources(${TARGET_NAME}
+  PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/errorindicators.cpp
+)

palace/utils/errorindicators.cpp → palace/fem/errorindicators.cpp

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "errorindicators.hpp"
-
+#include <fmt/os.h>
 #include "utils/communication.hpp"
 
 namespace palace
@@ -48,5 +48,4 @@ void ErrorIndicators::AddIndicators(const ErrorIndicators &error_indicators)
     (*this) = error_indicators;
   }
 }
-
 }  // namespace palace

palace/utils/errorindicators.hpp → palace/fem/errorindicators.hpp

@@ -1,8 +1,8 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-#ifndef PALACE_UTILS_ERROR_INDICATORS_HPP
-#define PALACE_UTILS_ERROR_INDICATORS_HPP
+#ifndef PALACE_FEM_ERROR_INDICATORS_HPP
+#define PALACE_FEM_ERROR_INDICATORS_HPP
 
 #include "linalg/vector.hpp"
 #include "utils/communication.hpp"
@@ -81,4 +81,4 @@ class ErrorIndicators
 
 }  // namespace palace
 
-#endif  // PALACE_UTILS_ERROR_INDICATORS_HPP
+#endif  // PALACE_FEM_ERROR_INDICATORS_HPP

palace/linalg/errorestimator.cpp

@@ -4,6 +4,7 @@
 #include "errorestimator.hpp"
 #include <limits>
 #include "fem/coefficient.hpp"
+#include "fem/errorindicators.hpp"
 #include "fem/integrator.hpp"
 #include "fem/multigrid.hpp"
 #include "linalg/amg.hpp"
@@ -12,7 +13,6 @@
 #include "linalg/rap.hpp"
 #include "models/materialoperator.hpp"
 #include "utils/communication.hpp"
-#include "utils/errorindicators.hpp"
 #include "utils/iodata.hpp"
 #include "utils/timer.hpp"
 
@@ -162,10 +162,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v
       }
     }
   }
-  for (auto &e : estimates)
-  {
-    e = std::sqrt(e);
-  }
+  linalg::CwiseSqrt(estimates);
 
   Mpi::GlobalSum(1, &normalization, nd_fespace.GetComm());
   normalization = std::sqrt(normalization);
@@ -306,10 +303,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const
       paraview.Save();
     }
   }
-  for (auto &e : estimates)
-  {
-    e = std::sqrt(e);
-  }
+  linalg::CwiseSqrt(estimates);
 
   Mpi::GlobalSum(1, &normalization, h1_fespace.GetComm());
   normalization = std::sqrt(normalization);

palace/linalg/errorestimator.hpp

@@ -6,10 +6,10 @@
 
 #include <memory>
 #include <mfem.hpp>
+#include "fem/errorindicators.hpp"
 #include "linalg/ksp.hpp"
 #include "linalg/operator.hpp"
 #include "linalg/vector.hpp"
-#include "utils/errorindicators.hpp"
 
 namespace palace
 {

palace/linalg/vector.cpp

@@ -356,6 +356,13 @@ void SetRandomSign(MPI_Comm comm, ComplexVector &x, int seed)
                { XI[i] = (XI[i] > 0.0) ? 1.0 : ((XI[i] < 0.0) ? -1.0 : 0.0); });
 }
 
+void CwiseSqrt(Vector &x)
+{
+  const int N = x.Size();
+  auto *X = x.ReadWrite();
+  mfem::forall(N, [=] MFEM_HOST_DEVICE(int i) { X[i] = std::sqrt(X[i]); });
+}
+
 template <>
 void SetSubVector(Vector &x, const mfem::Array<int> &rows, double s)
 {

palace/linalg/vector.hpp

@@ -139,6 +139,9 @@ void SetRandomReal(MPI_Comm comm, VecType &x, int seed = 0);
 template <typename VecType>
 void SetRandomSign(MPI_Comm comm, VecType &x, int seed = 0);
 
+// Component-wise square root.
+void CwiseSqrt(Vector &x);
+
 // Calculate the inner product yᴴ x or yᵀ x.
 template <typename VecType>
 inline auto Dot(MPI_Comm comm, const VecType &x, const VecType &y)

palace/main.cpp

@@ -12,9 +12,9 @@
 #include "drivers/electrostaticsolver.hpp"
 #include "drivers/magnetostaticsolver.hpp"
 #include "drivers/transientsolver.hpp"
+#include "fem/errorindicators.hpp"
 #include "linalg/slepc.hpp"
 #include "utils/communication.hpp"
-#include "utils/errorindicators.hpp"
 #include "utils/geodata.hpp"
 #include "utils/iodata.hpp"
 #include "utils/timer.hpp"

palace/models/postoperator.cpp

@@ -4,6 +4,7 @@
 #include "postoperator.hpp"
 
 #include "fem/coefficient.hpp"
+#include "fem/errorindicators.hpp"
 #include "models/curlcurloperator.hpp"
 #include "models/laplaceoperator.hpp"
 #include "models/lumpedportoperator.hpp"
@@ -12,7 +13,6 @@
 #include "models/surfacecurrentoperator.hpp"
 #include "models/waveportoperator.hpp"
 #include "utils/communication.hpp"
-#include "utils/errorindicators.hpp"
 #include "utils/geodata.hpp"
 #include "utils/iodata.hpp"
 

palace/utils/CMakeLists.txt

@@ -8,7 +8,6 @@
 target_sources(${TARGET_NAME}
   PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}/configfile.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/errorindicators.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/geodata.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/iodata.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/meshio.cpp