chore(revert): "chore(ci): don't use redirected earthly" (#6062)

This didn't end up causing issues, and is reasonably useful

Also uses a check for workflow consistency

.github/workflows/ci.yml

@@ -48,7 +48,7 @@ jobs:
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
-        run: earthly ./yarn-project+export-end-to-end
+        run: earthly-ci ./yarn-project+export-end-to-end
       # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
       # (Note ARM uses just 2 tests as a smoketest)
       - name: Create list of end-to-end jobs
@@ -76,7 +76,7 @@ jobs:
       - name: Test
         working-directory: ./yarn-project/end-to-end/
         timeout-minutes: 25
-        run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache
+        run: earthly-ci -P --no-output +${{ matrix.test }} --e2e_mode=cache
       # TODO
       # - name: Upload logs
       #   run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log
@@ -101,7 +101,7 @@ jobs:
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 25
         # limit our parallelism to half our cores
-        run: earthly --no-output +test --hardware_concurrency=64
+        run: earthly-ci --no-output +test --hardware_concurrency=64
 
   noir-projects:
     needs: setup
@@ -117,7 +117,7 @@ jobs:
           concurrency_key: noir-projects-${{ inputs.username || github.actor }}-x86
       - name: "Noir Projects"
         timeout-minutes: 25
-        run: earthly --no-output ./noir-projects/+test
+        run: earthly-ci --no-output ./noir-projects/+test
 
   yarn-project-formatting:
     needs: setup
@@ -134,7 +134,7 @@ jobs:
           concurrency_key: yarn-project-formatting-${{ github.actor }}-x86
       - name: "Yarn Project Tests"
         timeout-minutes: 25
-        run: earthly --no-output ./yarn-project/+format-check
+        run: earthly-ci --no-output ./yarn-project/+format-check
 
   yarn-project-test:
     needs: noir-projects
@@ -151,7 +151,7 @@ jobs:
           concurrency_key: yarn-project-test-${{ github.actor }}-x86
       - name: "Yarn Project Tests"
         timeout-minutes: 25
-        run: earthly --no-output ./yarn-project/+test
+        run: earthly-ci --no-output ./yarn-project/+test
 
   # push benchmarking binaries to dockerhub registry
   bb-bench-binaries:
@@ -169,7 +169,7 @@ jobs:
       - name: Build and Push Binaries
         timeout-minutes: 15
         working-directory: ./barretenberg/cpp/
-        run: earthly --push +bench-binaries
+        run: earthly-ci --push +bench-binaries
 
   setup-bench:
     uses: ./.github/workflows/setup-runner.yml
@@ -200,12 +200,12 @@ jobs:
       - name: Client IVC Bench
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 15
-        run: earthly --no-output +bench-client-ivc --bench_mode=cache
+        run: earthly-ci --no-output +bench-client-ivc --bench_mode=cache
 
       - name: Ultrahonk Bench
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 15
-        run: earthly --no-output +bench-ultra-honk --bench_mode=cache
+        run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache
 
   merge-check:
     runs-on: ubuntu-latest

.github/workflows/setup-runner.yml

@@ -62,6 +62,30 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
+      - name: Checkout Merge Pipeline Files
+        uses: actions/checkout@v4
+        # Only check PRs for consistency (not master)
+        if: ${{ github.event.pull_request.head.sha != '' }}
+        with:
+          path: merge-commit-pipeline-files
+          sparse-checkout: |
+            .github/workflows/ci.yml
+            .github/workflows/setup-runner.yml
+
+      - name: Ensure CI Consistency
+        # Only check PRs for consistency (not master)
+        if: ${{ github.event.pull_request.head.sha != '' }}
+        run: |
+          # Compare the checked-out CI configuration files with the reference files
+          if ! git diff --no-index .github/workflows/ci.yml merge-commit-pipeline-files/.github/workflows/ci.yml; then
+            echo "Error: ci.yml changes in master (or PR base). Please merge these changes."
+            exit 1
+          fi
+          if ! git diff --no-index .github/workflows/setup-runner.yml merge-commit-pipeline-files/.github/workflows/setup-runner.yml; then
+            echo "Error: setup-runner.yml changes in master (or PR base). Please merge these changes."
+            exit 1
+          fi
+
       - name: Start EC2 runner
         uses: ./.github/spot-runner-action
         with:

scripts/earthly-ci

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# A wrapper for Earthly that is meant to caught signs of known intermittent failures and continue.
+# The silver lining is if Earthly does crash, the cache can pick up the build.
+set -eu -o pipefail
+
+# Flag to determine if -i is present
+INTERACTIVE=false
+# Check for -i flag in the arguments
+for arg in "$@"; do
+    if [ "$arg" == "-i" ] || [ "$arg" == "--interactive" ]; then
+        INTERACTIVE=true
+        break
+    fi
+done
+
+OUTPUT_FILE=$(mktemp)
+# capture output to handle earthly edge cases
+if $INTERACTIVE ; then
+  # don't play nice with tee if interactive
+  earthly $@
+elif ! earthly $@ 2>&1 | tee $OUTPUT_FILE >&2 ; then
+  # we try earthly once, capturing output
+  # if we get one of our (unfortunate) known failures, handle retries
+  # TODO potentially handle other intermittent errors here
+  if grep 'failed to get edge: inconsistent graph state' $OUTPUT_FILE >/dev/null ; then
+    # TODO when earthly is overloaded we sometimes get
+    # 'failed to solve: failed to get edge: inconsistent graph state'
+    echo "Got 'inconsistent graph state'. Restarting earthly. See https://github.com/earthly/earthly/issues/2454'"
+    earthly $@
+    # TODO handle
+    # could not configure satellite: failed getting org: unable to authenticate: failed to execute login request: Post
+  else
+    # otherwise, propagate error
+    exit 1
+  fi
+fi