Add documentation and function to make debugging multiple MPI ranks easier

docs/DevGuide/DebuggingTips.md

@@ -7,6 +7,9 @@ See LICENSE.txt for details.
 
 Learn how to use a debugger such as gdb.
 
+You can debug MPI executables using the `sys::attach_debugger()` function. See
+the documentation of that function for details.
+
 # Useful gdb commands
 
 - To break when an exception is thrown `catch throw`

src/Evolution/Executables/GeneralizedHarmonic/EvolveGhBinaryBlackHole.cpp

@@ -13,11 +13,13 @@
 #include "Evolution/Systems/GeneralizedHarmonic/ConstraintDamping/RegisterDerivedWithCharm.hpp"
 #include "Parallel/CharmMain.tpp"
 #include "Utilities/Serialization/RegisterDerivedClassesWithCharm.hpp"
+#include "Utilities/System/AttachDebugger.hpp"
 
 extern "C" void CkRegisterMainModule() {
   Parallel::charmxx::register_main_module<EvolutionMetavars>();
   Parallel::charmxx::register_init_node_and_proc(
-      {&domain::creators::register_derived_with_charm,
+      {&sys::attach_debugger,
+       &domain::creators::register_derived_with_charm,
        &domain::creators::time_dependence::register_derived_with_charm,
        &domain::FunctionsOfTime::register_derived_with_charm,
        &gh::BoundaryCorrections::register_derived_with_charm,

src/Utilities/System/AttachDebugger.cpp

@@ -0,0 +1,34 @@
+// Distributed under the MIT License.
+// See LICENSE.txt for details.
+
+#include "Utilities/System/AttachDebugger.hpp"
+
+#include <chrono>
+#include <cstdlib>
+#include <iostream>
+#include <thread>
+#include <unistd.h>
+
+namespace sys {
+void attach_debugger() {
+  const char* env_enable_parallel_debug =
+      // NOLINTNEXTLINE(concurrency-mt-unsafe)
+      std::getenv("SPECTRE_ATTACH_DEBUGGER");
+  if (env_enable_parallel_debug != nullptr) {
+    // NOLINTNEXTLINE(modernize-avoid-c-arrays)
+    char hostname[2048];
+    gethostname(static_cast<char*>(hostname), sizeof(hostname));
+
+    const std::string output_info =
+        std::string{"pid:"} + std::to_string(getpid()) + std::string{":host:"} +
+        std::string{static_cast<char*>(hostname)} + "\n";
+    std::cout << output_info << std::flush;
+    // NOLINTNEXTLINE(misc-const-correctness)
+    volatile int i = 10;
+    while (i == 10) {
+      using namespace std::chrono_literals;
+      std::this_thread::sleep_for(std::chrono::seconds{i});
+    }
+  }
+}
+}  // namespace sys

src/Utilities/System/AttachDebugger.hpp

@@ -0,0 +1,28 @@
+// Distributed under the MIT License.
+// See LICENSE.txt for details.
+
+#pragma once
+
+namespace sys {
+/*!
+ * \brief Provide an infinite loop to attach a debugger during startup. Useful
+ * for debugging MPI runs.
+ *
+ * Each MPI rank writes a file name `spectre_pid_#_host_NAME` to the working
+ * directory. This allows you to attach GDB to the running process using
+ * `gdb --pid=PID`, once for each MPI rank. You must then halt the program
+ * using `C-c` and then call `set var i = 7` inside GDB. Once you've done this
+ * on each MPI rank, you can have each MPI rank `continue`.
+ *
+ * To add support for attaching to a debugger in an executable, you must add
+ * `sys::attach_debugger` to the
+ * `Parallel::charmxx::register_init_node_and_proc` init node functions. Then,
+ * when you launch the executable launch it as
+ * ```shell
+ * SPECTRE_ATTACH_DEBUGGER=1 mpirun -np N ...
+ * ```
+ * The environment variable `SPECTRE_ATTACH_DEBUGGER` being set tells the code
+ * to allow attaching from a debugger.
+ */
+void attach_debugger();
+}  // namespace sys

src/Utilities/System/CMakeLists.txt

@@ -9,6 +9,7 @@ spectre_target_sources(
   ${LIBRARY}
   PRIVATE
   Abort.cpp
+  AttachDebugger.cpp
   Exit.cpp
   ParallelInfo.cpp
   Prefetch.cpp
@@ -19,6 +20,7 @@ spectre_target_headers(
   INCLUDE_DIRECTORY ${CMAKE_SOURCE_DIR}/src
   HEADERS
   Abort.hpp
+  AttachDebugger.hpp
   Exit.hpp
   ParallelInfo.hpp
   Prefetch.hpp

tools/CheckFiles.sh

@@ -30,6 +30,7 @@ iostream() {
     is_c++ "$1" \
      && whitelist "$1" \
                   'tests/Unit/IO/Exporter/BundledExporter/Test_BundledExporter.cpp$' \
+                  'src/Utilities/System/AttachDebugger.cpp' \
      && grep -q '#include <iostream>' "$1"
 }
 iostream_report() {