From 26cba9e4ef9b63bf100e451b66cfe3ea62ab416c Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 26 Apr 2024 11:01:56 -0500 Subject: [PATCH] chore(revert): "chore(ci): don't use redirected earthly" (#6062) This didn't end up causing issues, and is reasonably useful Also uses a check for workflow consistency --- .github/workflows/ci.yml | 18 +++++++-------- .github/workflows/setup-runner.yml | 24 ++++++++++++++++++++ scripts/earthly-ci | 36 ++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 9 deletions(-) create mode 100755 scripts/earthly-ci diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc694cb8c1d..3f8f232f57a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: # prepare images locally, tagged by commit hash - name: "Build E2E Image" timeout-minutes: 40 - run: earthly ./yarn-project+export-end-to-end + run: earthly-ci ./yarn-project+export-end-to-end # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end # (Note ARM uses just 2 tests as a smoketest) - name: Create list of end-to-end jobs @@ -76,7 +76,7 @@ jobs: - name: Test working-directory: ./yarn-project/end-to-end/ timeout-minutes: 25 - run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + run: earthly-ci -P --no-output +${{ matrix.test }} --e2e_mode=cache # TODO # - name: Upload logs # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log @@ -101,7 +101,7 @@ jobs: working-directory: ./barretenberg/cpp/ timeout-minutes: 25 # limit our parallelism to half our cores - run: earthly --no-output +test --hardware_concurrency=64 + run: earthly-ci --no-output +test --hardware_concurrency=64 noir-projects: needs: setup @@ -117,7 +117,7 @@ jobs: concurrency_key: noir-projects-${{ inputs.username || github.actor }}-x86 - name: "Noir Projects" timeout-minutes: 25 - run: earthly --no-output ./noir-projects/+test + run: earthly-ci --no-output ./noir-projects/+test yarn-project-formatting: needs: setup @@ -134,7 +134,7 @@ jobs: concurrency_key: yarn-project-formatting-${{ github.actor }}-x86 - name: "Yarn Project Tests" timeout-minutes: 25 - run: earthly --no-output ./yarn-project/+format-check + run: earthly-ci --no-output ./yarn-project/+format-check yarn-project-test: needs: noir-projects @@ -151,7 +151,7 @@ jobs: concurrency_key: yarn-project-test-${{ github.actor }}-x86 - name: "Yarn Project Tests" timeout-minutes: 25 - run: earthly --no-output ./yarn-project/+test + run: earthly-ci --no-output ./yarn-project/+test # push benchmarking binaries to dockerhub registry bb-bench-binaries: @@ -169,7 +169,7 @@ jobs: - name: Build and Push Binaries timeout-minutes: 15 working-directory: ./barretenberg/cpp/ - run: earthly --push +bench-binaries + run: earthly-ci --push +bench-binaries setup-bench: uses: ./.github/workflows/setup-runner.yml @@ -200,12 +200,12 @@ jobs: - name: Client IVC Bench working-directory: ./barretenberg/cpp/ timeout-minutes: 15 - run: earthly --no-output +bench-client-ivc --bench_mode=cache + run: earthly-ci --no-output +bench-client-ivc --bench_mode=cache - name: Ultrahonk Bench working-directory: ./barretenberg/cpp/ timeout-minutes: 15 - run: earthly --no-output +bench-ultra-honk --bench_mode=cache + run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache merge-check: runs-on: ubuntu-latest diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml index 748ab5edeab..a29ccc6d5e5 100644 --- a/.github/workflows/setup-runner.yml +++ b/.github/workflows/setup-runner.yml @@ -62,6 +62,30 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} + - name: Checkout Merge Pipeline Files + uses: actions/checkout@v4 + # Only check PRs for consistency (not master) + if: ${{ github.event.pull_request.head.sha != '' }} + with: + path: merge-commit-pipeline-files + sparse-checkout: | + .github/workflows/ci.yml + .github/workflows/setup-runner.yml + + - name: Ensure CI Consistency + # Only check PRs for consistency (not master) + if: ${{ github.event.pull_request.head.sha != '' }} + run: | + # Compare the checked-out CI configuration files with the reference files + if ! git diff --no-index .github/workflows/ci.yml merge-commit-pipeline-files/.github/workflows/ci.yml; then + echo "Error: ci.yml changes in master (or PR base). Please merge these changes." + exit 1 + fi + if ! git diff --no-index .github/workflows/setup-runner.yml merge-commit-pipeline-files/.github/workflows/setup-runner.yml; then + echo "Error: setup-runner.yml changes in master (or PR base). Please merge these changes." + exit 1 + fi + - name: Start EC2 runner uses: ./.github/spot-runner-action with: diff --git a/scripts/earthly-ci b/scripts/earthly-ci new file mode 100755 index 00000000000..43eeb9b17aa --- /dev/null +++ b/scripts/earthly-ci @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# A wrapper for Earthly that is meant to caught signs of known intermittent failures and continue. +# The silver lining is if Earthly does crash, the cache can pick up the build. +set -eu -o pipefail + +# Flag to determine if -i is present +INTERACTIVE=false +# Check for -i flag in the arguments +for arg in "$@"; do + if [ "$arg" == "-i" ] || [ "$arg" == "--interactive" ]; then + INTERACTIVE=true + break + fi +done + +OUTPUT_FILE=$(mktemp) +# capture output to handle earthly edge cases +if $INTERACTIVE ; then + # don't play nice with tee if interactive + earthly $@ +elif ! earthly $@ 2>&1 | tee $OUTPUT_FILE >&2 ; then + # we try earthly once, capturing output + # if we get one of our (unfortunate) known failures, handle retries + # TODO potentially handle other intermittent errors here + if grep 'failed to get edge: inconsistent graph state' $OUTPUT_FILE >/dev/null ; then + # TODO when earthly is overloaded we sometimes get + # 'failed to solve: failed to get edge: inconsistent graph state' + echo "Got 'inconsistent graph state'. Restarting earthly. See https://github.com/earthly/earthly/issues/2454'" + earthly $@ + # TODO handle + # could not configure satellite: failed getting org: unable to authenticate: failed to execute login request: Post + else + # otherwise, propagate error + exit 1 + fi +fi