Merge pull request #5546 from nilsvu/bbh_pipeline

Add skeleton BBH pipeline

containers/Dockerfile.buildenv

@@ -139,7 +139,8 @@ RUN apt-get update -y \
     && apt-get install -y cmake clang lld wget
 # Install build dependencies for the target architecture
 RUN xx-apt-get update -y \
-    && xx-apt-get install -y libc6-dev libstdc++-10-dev bison flex
+    && xx-apt-get install -y --no-install-recommends \
+        libc6-dev libstdc++-10-dev bison flex
 # - Ccache
 RUN wget https://github.com/ccache/ccache/releases/download/v4.8.2/ccache-4.8.2.tar.gz -O ccache.tar.gz \
     && tar -xzf ccache.tar.gz \

src/Domain/CoordinateMaps/Composition.cpp

@@ -388,7 +388,7 @@ GENERATE_INSTANTIATIONS(INSTANTIATE, (1, 2, 3))
 
 #undef INSTANTIATE
 
-#if defined(__clang__) && __clang_major__ >= 16
+#if defined(__clang__) && __clang_major__ >= 15
 #define INSTANTIATE2(_, data)                                                  \
   template domain::CoordinateMaps::Composition<                                \
       brigand::list<Frame::ElementLogical, Frame::BlockLogical,                \

support/CMakeLists.txt

@@ -1,4 +1,5 @@
 # Distributed under the MIT License.
 # See LICENSE.txt for details.
 
+add_subdirectory(Pipelines)
 add_subdirectory(Python)

support/Pipelines/Bbh/CMakeLists.txt

@@ -0,0 +1,25 @@
+# Distributed under the MIT License.
+# See LICENSE.txt for details.
+
+spectre_python_add_module(
+  Bbh
+  MODULE_PATH Pipelines
+  PYTHON_FILES
+  __init__.py
+  InitialData.py
+  InitialData.yaml
+  Inspiral.py
+  Inspiral.yaml
+  Ringdown.py
+  Ringdown.yaml
+)
+
+# Create a target to compile all executables for this pipeline
+set(PIPELINE_TARGET bbh)
+add_custom_target(${PIPELINE_TARGET})
+add_dependencies(
+  ${PIPELINE_TARGET}
+  EvolveGhBinaryBlackHole
+  EvolveGhSingleBlackHole
+  SolveXcts
+  )

support/Pipelines/Bbh/InitialData.py

@@ -0,0 +1,151 @@
+# Distributed under the MIT License.
+# See LICENSE.txt for details.
+
+import logging
+from pathlib import Path
+from typing import Union
+
+import click
+from rich.pretty import pretty_repr
+
+from spectre.support.Schedule import schedule, scheduler_options
+
+logger = logging.getLogger(__name__)
+
+ID_INPUT_FILE_TEMPLATE = Path(__file__).parent / "InitialData.yaml"
+
+
+def id_parameters(
+    mass_ratio: float,
+    separation: float,
+    orbital_angular_velocity: float,
+    refinement_level: int,
+    polynomial_order: int,
+):
+    """Determine initial data parameters from options.
+
+    These parameters fill the 'ID_INPUT_FILE_TEMPLATE'.
+
+    Arguments:
+      mass_ratio: Defined as q = M_A / M_B >= 1.
+      separation: Coordinate separation D of the black holes.
+      orbital_angular_velocity: Omega_0.
+      refinement_level: h-refinement level.
+      polynomial_order: p-refinement level.
+    """
+
+    # Sanity checks
+    assert mass_ratio >= 1.0, "Mass ratio is defined to be >= 1.0."
+
+    # Determine initial data parameters from options
+    M_A = mass_ratio / (1.0 + mass_ratio)
+    M_B = 1.0 / (1.0 + mass_ratio)
+    x_A = separation / (1.0 + mass_ratio)
+    x_B = x_A - separation
+    return {
+        "MassRight": M_A,
+        "MassLeft": M_B,
+        "XRight": x_A,
+        "XLeft": x_B,
+        "ExcisionRadiusRight": 0.89 * 2.0 * M_A,
+        "ExcisionRadiusLeft": 0.89 * 2.0 * M_B,
+        "OrbitalAngularVelocity": orbital_angular_velocity,
+        # Resolution
+        "L": refinement_level,
+        "P": polynomial_order,
+    }
+
+
+def generate_id(
+    mass_ratio: float,
+    separation: float,
+    orbital_angular_velocity: float,
+    refinement_level: int,
+    polynomial_order: int,
+    id_input_file_template: Union[str, Path] = ID_INPUT_FILE_TEMPLATE,
+    **scheduler_kwargs,
+):
+    """Generate initial data for a BBH simulation.
+
+    Parameters for the initial data will be inserted into the
+    'id_input_file_template'. The remaining options are forwarded to the
+    'schedule' command. See 'schedule' docs for details.
+    """
+    logger.warning(
+        "The BBH pipeline is still experimental. Please review the"
+        " generated input files."
+    )
+
+    # Determine initial data parameters from options
+    id_params = id_parameters(
+        mass_ratio=mass_ratio,
+        separation=separation,
+        orbital_angular_velocity=orbital_angular_velocity,
+        refinement_level=refinement_level,
+        polynomial_order=polynomial_order,
+    )
+    logger.debug(f"Initial data parameters: {pretty_repr(id_params)}")
+
+    # Schedule!
+    return schedule(id_input_file_template, **id_params, **scheduler_kwargs)
+
+
+@click.command(name="generate-id", help=generate_id.__doc__)
+@click.option(
+    "--mass-ratio",
+    "-q",
+    type=float,
+    help="Mass ratio of the binary, defined as q = M_A / M_B >= 1.",
+    required=True,
+)
+@click.option(
+    "--separation",
+    "-D",
+    type=float,
+    help="Coordinate separation D of the black holes.",
+    required=True,
+)
+@click.option(
+    "--orbital-angular-velocity",
+    "-w",
+    type=float,
+    help="Orbital angular velocity Omega_0.",
+    required=True,
+)
+@click.option(
+    "--refinement-level",
+    "-L",
+    type=int,
+    help="h-refinement level.",
+    default=0,
+    show_default=True,
+)
+@click.option(
+    "--polynomial-order",
+    "-P",
+    type=int,
+    help="p-refinement level.",
+    default=5,
+    show_default=True,
+)
+@click.option(
+    "--id-input-file-template",
+    type=click.Path(
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        readable=True,
+        path_type=Path,
+    ),
+    default=ID_INPUT_FILE_TEMPLATE,
+    help="Input file template for the initial data.",
+    show_default=True,
+)
+@scheduler_options
+def generate_id_command(**kwargs):
+    _rich_traceback_guard = True  # Hide traceback until here
+    generate_id(**kwargs)
+
+
+if __name__ == "__main__":
+    generate_id_command(help_option_names=["-h", "--help"])

support/Pipelines/Bbh/InitialData.yaml

@@ -0,0 +1,161 @@
+# Distributed under the MIT License.
+# See LICENSE.txt for details.
+
+Executable: SolveXcts
+
+---
+
+Background: &background
+  Binary:
+    XCoords: [&x_left {{ XLeft }}, &x_right {{ XRight }}]
+    ObjectLeft: &kerr_left
+      KerrSchild:
+        Mass: {{ MassLeft }}
+        Spin: [0., 0., 0.]
+        Center: [0., 0., 0.]
+    ObjectRight: &kerr_right
+      KerrSchild:
+        Mass: {{ MassRight }}
+        Spin: [0., 0., 0.]
+        Center: [0., 0., 0.]
+    AngularVelocity: {{ OrbitalAngularVelocity }}
+    Expansion: 0.
+    LinearVelocity: [0., 0., 0.]
+    FalloffWidths: [4.8, 4.8]
+
+InitialGuess: *background
+
+DomainCreator:
+  BinaryCompactObject:
+    ObjectA:
+      InnerRadius: {{ ExcisionRadiusRight }}
+      OuterRadius: 4.
+      XCoord: *x_right
+      Interior:
+        ExciseWithBoundaryCondition:
+          ApparentHorizon:
+            Center: [*x_right, 0., 0.]
+            Rotation: [0., 0., 0.]
+            Lapse: *kerr_right
+            NegativeExpansion: *kerr_right
+      UseLogarithmicMap: True
+    ObjectB:
+      InnerRadius: {{ ExcisionRadiusLeft }}
+      OuterRadius: 4.
+      XCoord: *x_left
+      Interior:
+        ExciseWithBoundaryCondition:
+          ApparentHorizon:
+            Center: [*x_left, 0., 0.]
+            Rotation: [0., 0., 0.]
+            Lapse: *kerr_left
+            NegativeExpansion: *kerr_left
+      UseLogarithmicMap: True
+    Envelope:
+      Radius: &outer_shell_inner_radius 60.
+      RadialDistribution: Projective
+    OuterShell:
+      Radius: &outer_radius 1e4
+      RadialDistribution: &outer_shell_distribution Inverse
+      OpeningAngle: 120.0
+      BoundaryCondition: Flatness
+    UseEquiangularMap: True
+    InitialRefinement:
+      ObjectAShell:     [{{ L }}, {{ L }}, {{ L }}]
+      ObjectBShell:     [{{ L }}, {{ L }}, {{ L }}]
+      ObjectACube:      [{{ L }}, {{ L }}, {{ L }}]
+      ObjectBCube:      [{{ L }}, {{ L }}, {{ L }}]
+      Envelope:         [{{ L }}, {{ L }}, {{ L }}]
+      OuterShell:       [{{ L }}, {{ L }}, {{ L + 2}}]
+    # This p-refinement represents a crude manual optimization of the domain. We
+    # will need AMR to optimize the domain further.
+    InitialGridPoints:
+      ObjectAShell:   [{{ P + 1}}, {{ P + 1}}, {{ P + 5}}]
+      ObjectBShell:   [{{ P + 1}}, {{ P + 1}}, {{ P + 5}}]
+      ObjectACube:    [{{ P + 1}}, {{ P + 1}}, {{ P + 2}}]
+      ObjectBCube:    [{{ P + 1}}, {{ P + 1}}, {{ P + 2}}]
+      Envelope:       [{{ P + 1}}, {{ P + 1}}, {{ P + 1}}]
+      OuterShell:     [{{ P + 1}}, {{ P + 1}}, {{ P + 1}}]
+
+Discretization:
+  DiscontinuousGalerkin:
+    PenaltyParameter: 1.
+    Massive: True
+
+Observers:
+  VolumeFileName: "BbhVolume"
+  ReductionFileName: "BbhReductions"
+
+NonlinearSolver:
+  NewtonRaphson:
+    ConvergenceCriteria:
+      MaxIterations: 20
+      RelativeResidual: 1.e-10
+      AbsoluteResidual: 1.e-10
+    SufficientDecrease: 1.e-4
+    MaxGlobalizationSteps: 40
+    DampingFactor: 1.
+    Verbosity: Verbose
+
+LinearSolver:
+  Gmres:
+    ConvergenceCriteria:
+      MaxIterations: 100
+      RelativeResidual: 1.e-11
+      AbsoluteResidual: 1.e-11
+    Verbosity: Quiet
+
+  Multigrid:
+    Iterations: 1
+    MaxLevels: Auto
+    PreSmoothing: True
+    PostSmoothingAtBottom: True
+    Verbosity: Silent
+    OutputVolumeData: False
+
+  SchwarzSmoother:
+    MaxOverlap: 2
+    Iterations: 3
+    Verbosity: Silent
+    SubdomainSolver:
+      Gmres:
+        ConvergenceCriteria:
+          MaxIterations: 3
+          RelativeResidual: 1.e-4
+          AbsoluteResidual: 1.e-10
+        Verbosity: Silent
+        Restart: None
+        Preconditioner:
+          MinusLaplacian:
+            Solver:
+              ExplicitInverse:
+                WriteMatrixToFile: None
+            BoundaryConditions: Auto
+    SkipResets: True
+    ObservePerCoreReductions: False
+
+RadiallyCompressedCoordinates:
+  InnerRadius: *outer_shell_inner_radius
+  OuterRadius: *outer_radius
+  Compression: *outer_shell_distribution
+
+EventsAndTriggers:
+  - Trigger: HasConverged
+    Events:
+      - ObserveFields:
+          SubfileName: VolumeData
+          VariablesToObserve:
+            - ConformalFactor
+            - Lapse
+            - Shift
+            - ShiftExcess
+            - SpatialMetric
+            - ExtrinsicCurvature
+            - RadiallyCompressedCoordinates
+          InterpolateToMesh: None
+          CoordinatesFloatingPointType: Double
+          FloatingPointTypes: [Double]
+
+ResourceInfo:
+  AvoidGlobalProc0: false
+  Singletons: Auto