chore: workaround earthly flake

.github/workflows/ci.yml

@@ -44,8 +44,9 @@ jobs:
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
-        run: earthly ./yarn-project+export-end-to-end
+        run: earthly-ci ./yarn-project+export-end-to-end
       # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
+      # (Note ARM uses just 2 tests as a smoketest)
       - name: Create list of end-to-end jobs
         id: e2e_list
         run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' |  sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
@@ -68,7 +69,7 @@ jobs:
       - name: Test
         working-directory: ./yarn-project/end-to-end/
         timeout-minutes: 25
-        run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache
+        run: earthly-ci -P --no-output +${{ matrix.test }} --e2e_mode=cache
       # TODO
       # - name: Upload logs
       #   run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log
@@ -92,7 +93,7 @@ jobs:
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 25
         # limit our parallelism to half our cores
-        run: earthly --no-output +test --hardware_concurrency=64
+        run: earthly-ci --no-output +test --hardware_concurrency=64
 
   # push benchmarking binaries to dockerhub registry
   bb-bench-binaries:
@@ -108,7 +109,7 @@ jobs:
         if: ${{ github.event.inputs.just_start_spot != 'true' }}
         timeout-minutes: 15
         working-directory: ./barretenberg/cpp/
-        run: earthly --push +bench-binaries
+        run: earthly-ci --push +bench-binaries
 
   setup-bench:
     uses: ./.github/workflows/setup-runner.yml
@@ -136,12 +137,12 @@ jobs:
       - name: Client IVC Bench
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 15
-        run: earthly --no-output +bench-client-ivc --bench_mode=cache
+        run: earthly-ci --no-output +bench-client-ivc --bench_mode=cache
 
       - name: Ultrahonk Bench
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 15
-        run: earthly --no-output +bench-ultra-honk --bench_mode=cache
+        run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache
 
   merge-check:
     runs-on: ${{ github.actor }}-x86

scripts/earthly-ci

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# A wrapper for Earthly that is meant to caught signs of known intermittent failures and continue.
+# The silver lining is if Earthly does crash, the cache can pick up the buil.
+set -eu -o pipefail
+
+# Flag to determine if -i is present
+INTERACTIVE=false
+# Check for -i flag in the arguments
+for arg in "$@"; do
+    if [ "$arg" == "-i" ] || [ "$arg" == "--interactive" ]; then
+        INTERACTIVE=true
+        break
+    fi
+done
+
+OUTPUT_FILE=$(mktemp)
+# capture output to handle earthly edge cases
+if $INTERACTIVE ; then
+  # don't play nice with tee if interactive
+  earthly $@
+elif ! earthly $@ 2>&1 | tee $OUTPUT_FILE >&2 ; then
+  # we try earthly once, capturing output
+  # if we get one of our (unfortunate) known failures, handle retries
+  # TODO potentially handle other intermittent errors here
+  if grep 'failed to get edge: inconsistent graph state' $OUTPUT_FILE >/dev/null ; then
+    # TODO when earthly is overloaded we sometimes get
+    # 'failed to solve: failed to get edge: inconsistent graph state'
+    echo "Got 'inconsistent graph state'. Restarting earthly. See https://github.com/earthly/earthly/issues/2454'"
+    earthly $@
+    # TODO handle
+    # could not configure satellite: failed getting org: unable to authenticate: failed to execute login request: Post
+  else
+    # otherwise, propagate error
+    exit 1
+  fi
+fi

scripts/setup_env.sh

@@ -8,6 +8,6 @@ echo FORCE_COLOR=1 >> $GITHUB_ENV
 echo "Logging in to Docker..."
 echo $1 | docker login -u aztecprotocolci --password-stdin
 
-# Make earthly-cloud and earthly-cloud-bench scripts available
+# Make earthly-ci script available
 echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV
 echo "EARTHLY_CONFIG=$(git rev-parse --show-toplevel)/.github/earthly-ci-config.yml" >> $GITHUB_ENV