wip! soliloquizing bronchitis's

.github/workflows/pathogen-repo-build.yaml

@@ -160,14 +160,16 @@ jobs:
       # Need to run this after the build repo is cloned so that cloning the
       # build repo does not overwrite the .git dir and remove the extra support files
       # that we need from nextstrain/.github repo
-      - name: Checkout ${{ needs.workflow-context.outputs.repository }} (sha ${{ needs.workflow-context.outputs.sha }})
+      - &checkout-workflow-support
+        name: Checkout ${{ needs.workflow-context.outputs.repository }} (sha ${{ needs.workflow-context.outputs.sha }})
         uses: actions/checkout@v3
         with:
           repository: ${{ needs.workflow-context.outputs.repository }}
           ref: ${{ needs.workflow-context.outputs.sha }}
           path: ${{ env.NEXTSTRAIN_GITHUB_DIR }}
 
-      - name: Setup runtime ${{ inputs.runtime }}
+      - &setup-runtime
+        name: Setup runtime ${{ inputs.runtime }}
         uses: ./.git/nextstrain/.github/actions/setup-nextstrain-cli
         with:
           cli-version: ">=7.1.0"
@@ -233,3 +235,65 @@ jobs:
             logs/
             .snakemake/log/
             ${{ inputs.artifact-paths }}
+
+    outputs:
+      AWS_BATCH_JOB_ID: ${{ env.AWS_BATCH_JOB_ID }}
+
+  # Wait for up to 6 hours (the GitHub Actions job timeout) for the AWS Batch
+  # job to finish.
+  wait-1: &wait
+    needs: [run-build, workflow-context]
+    if: needs.run-build.outputs.AWS_BATCH_JOB_ID
+
+    # XXX FIXME: We should use a self-hosted runner to avoid eating up our
+    # org-wide concurrency quota on GitHub's runners.  For example, for about
+    # $100/yr (committed to, but not paid upfront) we could self-host a single,
+    # always-on runner on a t4g.small EC2 instance with 2 CPUs and 2GB of RAM.
+    # We could also use other auto-scaling solutions and likely reduce the cost
+    # further while increasing capacity.  Then we'd select for that with
+    # runs-on: labels and/or groups, e.g. with the default labels:
+    #
+    #   runs-on: [self-hosted, linux, ARM64]
+    #
+    #   -trs, 4 August 2023
+    runs-on: ubuntu-latest
+
+    steps:
+      # Uses needs.workflow-context.outputs
+      - *checkout-workflow-support
+      - *setup-runtime
+
+      # XXX TODO: This will reiterate the full logs emitted so far on every
+      # wait-N job, which is annoying, but ah well.  Avoiding that is more work
+      # than I care to spend right now for too little gain.  It's maybe more
+      # desirable this way anyhow: the full logs of the Batch job will be
+      # captured in GitHub Actions logs by the last successful wait-N job.
+      #   -trs, 4 August 2023
+      - name: Attach to AWS Batch job
+        env:
+          AWS_BATCH_JOB_ID: ${{ needs.run-build.outputs.AWS_BATCH_JOB_ID }}
+        run: |
+          nextstrain build \
+            --aws-batch \
+            --attach "$AWS_BATCH_JOB_ID" \
+            --no-download
+
+  # Wait for up to another 6 hours (hours 6–12) if the preceding wait-N job
+  # timed out.  Our heuristic for job timeout is when the job is cancelled but
+  # the workflow isn't.
+  wait-2:
+    needs: [wait-1, run-build, workflow-context]
+    if: needs.wait-1.result == 'cancelled' && !cancelled()
+    <<: *wait
+
+  # 12–18 hours
+  wait-3:
+    needs: [wait-2, run-build, workflow-context]
+    if: needs.wait-2.result == 'cancelled' && !cancelled()
+    <<: *wait
+
+  # 18–24 hours
+  wait-4:
+    needs: [wait-3, run-build, workflow-context]
+    if: needs.wait-3.result == 'cancelled' && !cancelled()
+    <<: *wait