Add LTS CMake Option in GenHarmBase Executables

sxs-collaboration · Jun 9, 2024 · bbe3fe9 · bbe3fe9
1 parent 57fc561
commit bbe3fe9
Show file tree

Hide file tree

Showing 9 changed files with 63 additions and 18 deletions.
diff --git a/src/Evolution/Executables/GeneralizedHarmonic/CMakeLists.txt b/src/Evolution/Executables/GeneralizedHarmonic/CMakeLists.txt
@@ -44,6 +44,7 @@ function(add_generalized_harmonic_executable_without_horizon DIM)
     ${EXECUTABLE}
     PRIVATE
     DIM=${DIM}
+    USE_LTS=false
     )
   target_link_libraries(${EXECUTABLE} PRIVATE ${LIBS_TO_LINK})
 endfunction()
@@ -68,6 +69,12 @@ target_link_libraries(
   ControlSystem
   Deadlock
   )
+target_compile_definitions(
+  ${SINGLE_BH_EXECUTABLE}
+  PRIVATE
+  USE_LTS=true
+  )
+
 
 # BBH
 set(BBH_EXECUTABLE EvolveGhBinaryBlackHole)

diff --git a/src/Evolution/Executables/GeneralizedHarmonic/EvolveGhAndCharacteristic.hpp b/src/Evolution/Executables/GeneralizedHarmonic/EvolveGhAndCharacteristic.hpp
@@ -67,11 +67,11 @@ class CProxy_GlobalCache;
 /// \endcond
 
 template <bool EvolveCcm>
-struct EvolutionMetavars : public GeneralizedHarmonicTemplateBase<3>,
+struct EvolutionMetavars : public GeneralizedHarmonicTemplateBase<3, false>,
                            public CharacteristicExtractDefaults<EvolveCcm> {
   static constexpr size_t volume_dim = 3;
   using cce_base = CharacteristicExtractDefaults<EvolveCcm>;
-  using gh_base = GeneralizedHarmonicTemplateBase<volume_dim>;
+  using gh_base = GeneralizedHarmonicTemplateBase<volume_dim, false>;
   using typename gh_base::initialize_initial_data_dependent_quantities_actions;
   using cce_boundary_component = Cce::GhWorldtubeBoundary<EvolutionMetavars>;
 

diff --git a/src/Evolution/Executables/GeneralizedHarmonic/EvolveGhNoBlackHole.cpp b/src/Evolution/Executables/GeneralizedHarmonic/EvolveGhNoBlackHole.cpp
@@ -14,7 +14,7 @@
 #include "Utilities/Serialization/RegisterDerivedClassesWithCharm.hpp"
 
 // Parameters chosen in CMakeLists.txt
-using metavariables = EvolutionMetavars<DIM>;
+using metavariables = EvolutionMetavars<DIM, USE_LTS>;
 
 extern "C" void CkRegisterMainModule() {
   Parallel::charmxx::register_main_module<metavariables>();

diff --git a/src/Evolution/Executables/GeneralizedHarmonic/EvolveGhNoBlackHole.hpp b/src/Evolution/Executables/GeneralizedHarmonic/EvolveGhNoBlackHole.hpp
@@ -22,10 +22,11 @@
 #include "Utilities/ErrorHandling/Error.hpp"
 #include "Utilities/ProtocolHelpers.hpp"
 
-template <size_t VolumeDim>
-struct EvolutionMetavars : public GeneralizedHarmonicTemplateBase<VolumeDim> {
+template <size_t VolumeDim, bool UseLts>
+struct EvolutionMetavars
+    : public GeneralizedHarmonicTemplateBase<VolumeDim, UseLts> {
   static constexpr size_t volume_dim = VolumeDim;
-  using gh_base = GeneralizedHarmonicTemplateBase<volume_dim>;
+  using gh_base = GeneralizedHarmonicTemplateBase<volume_dim, UseLts>;
   using typename gh_base::const_global_cache_tags;
   using typename gh_base::dg_registration_list;
   using initialization_actions =

diff --git a/src/Evolution/Executables/GeneralizedHarmonic/EvolveGhSingleBlackHole.cpp b/src/Evolution/Executables/GeneralizedHarmonic/EvolveGhSingleBlackHole.cpp
@@ -14,15 +14,18 @@
 #include "Parallel/CharmMain.tpp"
 #include "Utilities/Serialization/RegisterDerivedClassesWithCharm.hpp"
 
+// Parameters chosen in CMakeLists.txt
+using metavariables = EvolutionMetavars<USE_LTS>;
+
 extern "C" void CkRegisterMainModule() {
-  Parallel::charmxx::register_main_module<EvolutionMetavars>();
+  Parallel::charmxx::register_main_module<metavariables>();
   Parallel::charmxx::register_init_node_and_proc(
       {&domain::creators::register_derived_with_charm,
        &domain::creators::time_dependence::register_derived_with_charm,
        &domain::FunctionsOfTime::register_derived_with_charm,
        &gh::BoundaryCorrections::register_derived_with_charm,
        &gh::ConstraintDamping::register_derived_with_charm,
        &control_system::size::register_derived_with_charm,
-       &register_factory_classes_with_charm<EvolutionMetavars>},
+       &register_factory_classes_with_charm<metavariables>},
       {});
 }
diff --git a/src/Evolution/Executables/GeneralizedHarmonic/EvolveGhSingleBlackHole.hpp b/src/Evolution/Executables/GeneralizedHarmonic/EvolveGhSingleBlackHole.hpp
@@ -76,9 +76,10 @@
 #include "Utilities/PrettyType.hpp"
 #include "Utilities/ProtocolHelpers.hpp"
 
-struct EvolutionMetavars : public GeneralizedHarmonicTemplateBase<3> {
+template <bool UseLts>
+struct EvolutionMetavars : public GeneralizedHarmonicTemplateBase<3, UseLts> {
   static constexpr size_t volume_dim = 3;
-  using gh_base = GeneralizedHarmonicTemplateBase<volume_dim>;
+  using gh_base = GeneralizedHarmonicTemplateBase<volume_dim, UseLts>;
   using typename gh_base::initialize_initial_data_dependent_quantities_actions;
   using typename gh_base::system;
 
@@ -263,7 +264,7 @@ struct EvolutionMetavars : public GeneralizedHarmonicTemplateBase<3> {
         evolution::Actions::ProjectRunEventsAndDenseTriggers,
         ::amr::projectors::DefaultInitialize<
             Initialization::Tags::InitialTimeDelta,
-            Initialization::Tags::InitialSlabSize<local_time_stepping>,
+            Initialization::Tags::InitialSlabSize<gh_base::local_time_stepping>,
             ::domain::Tags::InitialExtents<volume_dim>,
             ::domain::Tags::InitialRefinementLevels<volume_dim>,
             evolution::dg::Tags::Quadrature,

diff --git a/src/Evolution/Executables/GeneralizedHarmonic/GeneralizedHarmonicBase.hpp b/src/Evolution/Executables/GeneralizedHarmonic/GeneralizedHarmonicBase.hpp
@@ -320,11 +320,12 @@ struct FactoryCreation : tt::ConformsTo<Options::protocols::FactoryCreation> {
 };
 }  // namespace detail
 
-template <size_t VolumeDim>
+template <size_t VolumeDim, bool LocalTimeStepping>
 struct GeneralizedHarmonicTemplateBase {
   static constexpr size_t volume_dim = VolumeDim;
   using system = gh::System<volume_dim>;
-  using TimeStepperBase = TimeStepper;
+  using TimeStepperBase =
+      tmpl::conditional_t<LocalTimeStepping, LtsTimeStepper, TimeStepper>;
 
   static constexpr bool local_time_stepping =
       TimeStepperBase::local_time_stepping;

diff --git a/support/Pipelines/Bbh/Ringdown.yaml b/support/Pipelines/Bbh/Ringdown.yaml
@@ -45,10 +45,26 @@ DomainCreator:
 
 Evolution:
   InitialTime: 0.0
-  InitialTimeStep: 0.01
+  InitialTimeStep: 0.0002
+  # This is the smallest interval we'd need to observe time step/constraints. If
+  # you need it smaller you can edit it, but make sure to change the slab
+  # intervals in the EventsAndTriggers
+  InitialSlabSize: 0.1
+  StepChoosers:
+    - Increase:
+        Factor: 2
+    - ElementSizeCfl:
+        SafetyFactor: 0.5
+    - ErrorControl:
+        AbsoluteTolerance: 1e-10
+        RelativeTolerance: 1e-8
+        MaxFactor: 2
+        MinFactor: 0.25
+        SafetyFactor: 0.95
+  # Found that order 4 offers a significant speedup compared to order 5
   TimeStepper:
     AdamsBashforth:
-      Order: 1
+      Order: 4
 
 EvolutionSystem:
   GeneralizedHarmonic:

diff --git a/tests/InputFiles/GeneralizedHarmonic/KerrSchild.yaml b/tests/InputFiles/GeneralizedHarmonic/KerrSchild.yaml
@@ -4,7 +4,7 @@
 Executable: EvolveGhSingleBlackHole
 Testing:
   Check: parse;execute_check_output
-  Timeout: 8
+  Timeout: 15
   Priority: High
 ExpectedOutput:
   - GhKerrSchildVolume0.h5
@@ -27,10 +27,26 @@ ResourceInfo:
 
 Evolution:
   InitialTime: 0.0
-  InitialTimeStep: 0.01
+  InitialTimeStep: 0.0002
+  # *Important* this slab size was chosen for testing purposes, a more realistic
+  # slab size for evolutions would be 0.1 to observe constraints. If you would
+  # like more frequent output, consider using dense output.
+  InitialSlabSize: 0.001
+  StepChoosers:
+    - Increase:
+        Factor: 2
+    - ElementSizeCfl:
+        SafetyFactor: 0.5
+    - ErrorControl:
+        AbsoluteTolerance: 1e-10
+        RelativeTolerance: 1e-8
+        MaxFactor: 2
+        MinFactor: 0.25
+        SafetyFactor: 0.95
+  # Found that order 4 offers a significant speedup compared to order 5
   TimeStepper:
     AdamsBashforth:
-      Order: 1
+      Order: 4
 
 Amr:
   Criteria: