Merge d8d040c into 096a0b2

.github/earthly-ci-config.yml

@@ -1,5 +1,5 @@
 global:
     cache_size_pct: 50
-    buildkit_max_parallelism: 5
+    buildkit_max_parallelism: 10
     container_frontend: docker-shell
 buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]

build-system/s3-cache-scripts/Earthfile

@@ -28,13 +28,10 @@ WRITE_CONTENT_HASH:
   LET local_dir=$(pwd | sed 's|^/usr/src/||')
   COPY (+compute-content-hash/.content-hash --local_directory_equivalent="$local_dir" --rebuild_patterns="$rebuild_patterns") .
 
-# 1) starts our cache server if not up
-# 2) copies necessary context
-# 3) first tries our rebuild pattern and context against our cache tool to see if we can use our cache
-# 4) if not, we build with $command
-# 5) then we try to upload with our cache tool for next time
-# note, if the cache tool is unreachable at $AZTEC_CACHE_TOOL_IP we silently fail cache-download-pattern.sh
-# and silently pass cache-upload-pattern.sh meaning the command always runs
+# 1) copies necessary context
+# 2) first tries our rebuild pattern and context against our S3/minio cache
+# 3) if not, we build with $command
+# 4) then we try to upload to S3/minio cache for next time
 WITH_CACHE:
   FUNCTION
   ARG command
@@ -46,6 +43,7 @@ WITH_CACHE:
   DO +SETUP_CACHE
   DO +WRITE_CONTENT_HASH --rebuild_patterns="$rebuild_patterns"
   # configure AWS access and S3 cache params (note these can be blank if not using S3 build cache)
+  # we abuse secrets here somewhat to keep a stable build hash in Earthly, as secrets don't affect earthly's caching
   RUN --secret AWS_ACCESS_KEY_ID \
       --secret AWS_SECRET_ACCESS_KEY \
       --secret S3_BUILD_CACHE_UPLOAD \

build-system/s3-cache-scripts/README.md

@@ -4,12 +4,7 @@ Assumes a git committed state. If that is not the case, you should not use the c
 Rationale:
 - We need a unified cache tool that can support distributed caching. This is needed to replace our old docker image-based caching. It is easier to share S3 access and overall easier to use S3 tarballs rather than docker images.
 
-We put a list of .rebuild_patterns files (that have regex matching files in git) into the AZTEC_CACHE_REBUILD_PATTERNS env variable and a prefix for our artifact.
-It does not need to be a unique name per build as our content hash will be appended.
-- AZTEC_CACHE_REBUILD_PATTERNS=barretenberg/cpp/.rebuild_patterns ./cache-download-pattern.sh barretenberg
-
-If we want to upload to the local or distributed cache (again depending if we started the server with S3_READ/S3_WRITE):
-- AZTEC_CACHE_REBUILD_PATTERNS=barretenberg/cpp/.rebuild_patterns ./cache-download-pattern.sh barretenberg
+See Earthfile and test.sh for usage.
 
 Installation:
-- This is just some shell scripts, but you do need AWS credentials set up and aws commandline installed otherwise the scripts **do nothing**.
+- This is just some shell scripts, but you do need AWS credentials set up and aws commandline installed otherwise the scripts **do nothing**.

build-system/s3-cache-scripts/compute-content-hash.sh

@@ -27,4 +27,5 @@ fi
 
 # Sort the hashes and compute the content hash
 CONTENT_HASH=$(echo "$FILE_HASHES" | sort | git hash-object --stdin)
-echo "$CONTENT_HASH"
+# important: include architecture in content hash because we target x86_64 and arm64
+echo "$CONTENT_HASH-$(uname -m)"

noir/Earthfile

@@ -27,7 +27,8 @@ nargo-src:
     # just only when it changes
     # the commit hash gets injected into version strings
     DO ../build-system/s3-cache-scripts/+WRITE_CONTENT_HASH --rebuild_patterns=".rebuild_patterns_native"
-    ENV COMMIT_HASH=$(cat .content-hash)
+    # We use our content hash as a fake commit hash, stripped of non-semver characters
+    ENV COMMIT_HASH=$(cat .content-hash | sed 's/[^a-zA-Z0-9.-]//g')
 
     # # borrow Nix's approach to build everything in 1970
     ENV SOURCE_TIMESTAMP=1
@@ -267,4 +268,4 @@ bench-publish-acir-bb:
   RUN mkdir -p ./log
   RUN docker run -v "$(pwd)/log":/log -e LOG_FILE=/log/bench-acir.jsonl --rm aztecprotocol/barretenberg-acir-benches:$AZTEC_DOCKER_TAG ./bench_acir_tests.sh
 
-  DO ../+UPLOAD_LOGS --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH
+  DO ../+UPLOAD_LOGS --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH

yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   fork:
-    image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-amd64
+    image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-${ARCH_TAG:-amd64}
     pull_policy: always
     entrypoint: >
       sh -c '

yarn-project/end-to-end/scripts/docker-compose-p2p.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   fork:
-    image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-amd64
+    image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-${ARCH_TAG:-amd64}
     pull_policy: always
     entrypoint: 'anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337'
     expose:

yarn-project/end-to-end/scripts/docker-compose-wallet.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   fork:
-    image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-amd64
+    image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-${ARCH_TAG:-amd64}
     pull_policy: always
     entrypoint: >
       sh -c '

yarn-project/end-to-end/scripts/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   fork:
-    image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-amd64
+    image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-${ARCH_TAG:-amd64}
     pull_policy: always
     entrypoint: >
       sh -c '

yarn-project/end-to-end/scripts/e2e_compose_test.sh

@@ -19,6 +19,15 @@ COMPOSE_FILE="${COMPOSE_FILE:-./scripts/docker-compose.yml}"
 export DEBUG="${DEBUG:-aztec:*}"
 export HARDWARE_CONCURRENCY="${HARDWARE_CONCURRENCY:-}"
 export AZTEC_DOCKER_TAG="${AZTEC_DOCKER_TAG:-$(git rev-parse HEAD)}"
+
+# Set the ARCH_TAG variable for docker compose
+ARCH=$(uname -m)
+if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "aarch64" ]; then
+    export ARCH_TAG="arm64"
+else
+    export ARCH_TAG="amd64"
+fi
+
 FORCE_BUILD="${FORCE_BUILD:-true}"
 
 # Compute project_name