chore: ebs attach robustness (#6108)

AztecProtocol · Apr 30, 2024 · c702688 · c702688
1 parent c985c73
commit c702688
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 3 deletions.
diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml
@@ -78,11 +78,11 @@ jobs:
         run: |
           # Compare the checked-out CI configuration files with the reference files
           if ! git diff --no-index .github/workflows/ci.yml merge-commit-pipeline-files/.github/workflows/ci.yml; then
-            echo "Error: ci.yml changes in master (or PR base). Please merge these changes."
+            echo "Error: ci.yml changes in master (or PR base). Please merge these changes. This is to prevent surprises from Github Action's merge behavior."
             exit 1
           fi
           if ! git diff --no-index .github/workflows/setup-runner.yml merge-commit-pipeline-files/.github/workflows/setup-runner.yml; then
-            echo "Error: setup-runner.yml changes in master (or PR base). Please merge these changes."
+            echo "Error: setup-runner.yml changes in master (or PR base). Please merge these changes. This is to prevent surprises from Github Action's merge behavior."
             exit 1
           fi
 

diff --git a/scripts/attach_ebs_cache.sh b/scripts/attach_ebs_cache.sh
@@ -14,8 +14,20 @@ if [ -f /run/.ebs-cache-mounted ] ; then
   WAIT_INTERVAL=10  # Interval between checks in seconds
   elapsed_time=0
   # Check for existing mount, assume we can continue if existing
+
   while ! mount | grep -q "/var/lib/docker type ext4"; do
-    echo "Someone already marked as mounting, waiting for them..."
+    echo "Someone already marked as mounting, terminating any stopped instances and waiting..."
+    # Identify and terminate instances in 'STOPPED' state that are using this volume
+    STOPPED_INSTANCES=$(aws ec2 describe-instances \
+      --region $REGION \
+      --filters "Name=instance-state-name,Values=stopped" "Name=block-device-mapping.volume-id,Values=$VOLUME_ID" \
+      --query "Reservations[*].Instances[*].InstanceId" \
+      --output text)
+
+    for instance in $STOPPED_INSTANCES; do
+      echo "Terminating instance $instance"
+      aws ec2 terminate-instances --instance-ids $instance
+    done
     if [ $elapsed_time -ge $MAX_WAIT_TIME ]; then
       echo "Cache mount did not become available within $MAX_WAIT_TIME seconds... race condition?"
       exit 1