fix: disk attach edge case (#6430)

AztecProtocol · May 15, 2024 · 2366ad3 · 2366ad3
1 parent c2e72b1
commit 2366ad3
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 4 deletions.
diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml
@@ -74,4 +74,8 @@ runs:
           [ ! -f ~/delay_shutdown.sh ] && ( ! [ -f ~/.user-data-started ] || [ -f ~/.user-data-finished ] ) && break
           [ -f ~/.setup-complete ] && ( ! [ -f ~/.user-data-started ] || [ -f ~/.user-data-finished ] ) && break
           sleep 1
-        done
+        done
+        if ! [ -f ~/.setup-complete ] && [ -f ~/.user-data-started ] ; then
+          echo "Cache mount not seeming to be complete, exiting!"
+          exit 1
+        fi
diff --git a/.github/spot-runner-action/dist/index.js b/.github/spot-runner-action/dist/index.js
@@ -1028,7 +1028,7 @@ class UserData {
                 `sudo service docker restart`,
                 "sudo wget -q https://github.com/earthly/earthly/releases/download/v0.8.10/earthly-linux-$(dpkg --print-architecture) -O /usr/local/bin/earthly",
                 "sudo chmod +x /usr/local/bin/earthly",
-                "sudo apt install -y brotli",
+                "for i in {1..3} ; do sudo apt install -y brotli && break; sleep 10; done",
                 'echo "MaxStartups 1000" >> /etc/ssh/sshd_config',
                 'echo "ClientAliveInterval=30" >> /etc/ssh/sshd_config',
                 'echo "ClientAliveCountMax=20" >> /etc/ssh/sshd_config',

diff --git a/.github/spot-runner-action/src/userdata.ts b/.github/spot-runner-action/src/userdata.ts
@@ -22,7 +22,7 @@ export class UserData {
       `sudo service docker restart`,
       "sudo wget -q https://github.com/earthly/earthly/releases/download/v0.8.10/earthly-linux-$(dpkg --print-architecture) -O /usr/local/bin/earthly",
       "sudo chmod +x /usr/local/bin/earthly",
-      "sudo apt install -y brotli",
+      "for i in {1..3} ; do sudo apt install -y brotli && break; sleep 10; done",
       'echo "MaxStartups 1000" >> /etc/ssh/sshd_config',
       'echo "ClientAliveInterval=30" >> /etc/ssh/sshd_config',
       'echo "ClientAliveCountMax=20" >> /etc/ssh/sshd_config',

diff --git a/scripts/ci/attach_ebs_cache.sh b/scripts/ci/attach_ebs_cache.sh
@@ -51,7 +51,12 @@ if [ -f ~/.ebs-cache-mounted ] ; then
     sleep 1
     echo "Waiting for other mount to finish."
   done
-  exit 0
+  if [ -f ~/.setup-complete ] ; then
+    echo "Failed to find mount! Taking mount lock and trying..."
+    rm -f ~/.ebs-cache-mounted
+  else
+    exit 0
+  fi
 fi
 
 # Mark to prevent race conditions