diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index 23116c899b9..415773944d6 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -74,4 +74,8 @@ runs: [ ! -f ~/delay_shutdown.sh ] && ( ! [ -f ~/.user-data-started ] || [ -f ~/.user-data-finished ] ) && break [ -f ~/.setup-complete ] && ( ! [ -f ~/.user-data-started ] || [ -f ~/.user-data-finished ] ) && break sleep 1 - done \ No newline at end of file + done + if ! [ -f ~/.setup-complete ] && [ -f ~/.user-data-started ] ; then + echo "Cache mount not seeming to be complete, exiting!" + exit 1 + fi diff --git a/.github/spot-runner-action/dist/index.js b/.github/spot-runner-action/dist/index.js index 6d390638488..76535852540 100644 --- a/.github/spot-runner-action/dist/index.js +++ b/.github/spot-runner-action/dist/index.js @@ -1028,7 +1028,7 @@ class UserData { `sudo service docker restart`, "sudo wget -q https://github.com/earthly/earthly/releases/download/v0.8.10/earthly-linux-$(dpkg --print-architecture) -O /usr/local/bin/earthly", "sudo chmod +x /usr/local/bin/earthly", - "sudo apt install -y brotli", + "for i in {1..3} ; do sudo apt install -y brotli && break; sleep 10; done", 'echo "MaxStartups 1000" >> /etc/ssh/sshd_config', 'echo "ClientAliveInterval=30" >> /etc/ssh/sshd_config', 'echo "ClientAliveCountMax=20" >> /etc/ssh/sshd_config', diff --git a/.github/spot-runner-action/src/userdata.ts b/.github/spot-runner-action/src/userdata.ts index 8ff78513791..976cad46eb8 100644 --- a/.github/spot-runner-action/src/userdata.ts +++ b/.github/spot-runner-action/src/userdata.ts @@ -22,7 +22,7 @@ export class UserData { `sudo service docker restart`, "sudo wget -q https://github.com/earthly/earthly/releases/download/v0.8.10/earthly-linux-$(dpkg --print-architecture) -O /usr/local/bin/earthly", "sudo chmod +x /usr/local/bin/earthly", - "sudo apt install -y brotli", + "for i in {1..3} ; do sudo apt install -y brotli && break; sleep 10; done", 'echo "MaxStartups 1000" >> /etc/ssh/sshd_config', 'echo "ClientAliveInterval=30" >> /etc/ssh/sshd_config', 'echo "ClientAliveCountMax=20" >> /etc/ssh/sshd_config', diff --git a/scripts/ci/attach_ebs_cache.sh b/scripts/ci/attach_ebs_cache.sh index 2ca88f3c602..95e89e4b67d 100755 --- a/scripts/ci/attach_ebs_cache.sh +++ b/scripts/ci/attach_ebs_cache.sh @@ -51,7 +51,12 @@ if [ -f ~/.ebs-cache-mounted ] ; then sleep 1 echo "Waiting for other mount to finish." done - exit 0 + if [ -f ~/.setup-complete ] ; then + echo "Failed to find mount! Taking mount lock and trying..." + rm -f ~/.ebs-cache-mounted + else + exit 0 + fi fi # Mark to prevent race conditions