feat: bootstrap_docker skips build it can pull from ecr.

.circleci/config.yml

@@ -376,7 +376,7 @@ jobs:
   yarn-project-base:
     machine:
       image: ubuntu-2204:2023.07.2
-    resource_class: xlarge
+    resource_class: large
     steps:
       - *checkout
       - *setup_env
@@ -395,27 +395,27 @@ jobs:
           name: Build
           command: build yarn-project | add_timestamps
 
-  yarn-project-format:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
+  yarn-project-formatting:
+    machine:
+      image: ubuntu-2204:2023.07.2
+    resource_class: large
     steps:
       - *checkout
       - *setup_env
       - run:
-          name: Build
-          command: cond_spot_run_script yarn-project 32 cond_run_container yarn-project formatting | add_timestamps
+          name: Check Formatting
+          command: cond_run_container yarn-project formatting | add_timestamps
 
-  yarn-project-test:
+  yarn-project-tests:
     docker:
       - image: aztecprotocol/alpine-build-image
     resource_class: small
     steps:
       - *checkout
       - *setup_env
       - run:
-          name: Build
-          command: cond_spot_run_script yarn-project 64 cond_run_container yarn-project test | add_timestamps
+          name: Test
+          command: cond_spot_run_container yarn-project 64 test | add_timestamps
 
   aztec-sandbox-base:
     machine:
@@ -1130,18 +1130,18 @@ workflows:
             - circuits-wasm-linux-clang
             - l1-contracts
           <<: *defaults
-      - yarn-project-format:
+      - yarn-project:
           requires:
-            - yarn-project
+            - yarn-project-base
+            - noir-contracts-build
           <<: *defaults
-      - yarn-project-test:
+      - yarn-project-formatting:
           requires:
             - yarn-project
           <<: *defaults
-      - yarn-project:
+      - yarn-project-tests:
           requires:
-            - yarn-project-base
-            - noir-contracts-build
+            - yarn-project
           <<: *defaults
 
       - end-to-end: *yarn_project

barretenberg/cpp/dockerfiles/Dockerfile.wasm-linux-clang

@@ -4,8 +4,11 @@ WORKDIR /usr/src/barretenberg/cpp
 COPY ./scripts/install-wasi-sdk.sh ./scripts/install-wasi-sdk.sh
 RUN ./scripts/install-wasi-sdk.sh
 COPY . .
-RUN cmake --preset wasm && cmake --build --preset wasm
-RUN cmake --preset wasm-threads && cmake --build --preset wasm-threads
+# Building both wasm's in parallel reduces build from 120s to 80s.
+RUN (cmake --preset wasm && cmake --build --preset wasm) & \
+    (cmake --preset wasm-threads && cmake --build --preset wasm-threads) & \
+    wait
+
 
 FROM scratch
 WORKDIR /usr/src/barretenberg/cpp

bootstrap_docker.sh

@@ -22,7 +22,6 @@
 set -e
 
 PROJECT_NAME=${1:-}
-COMMIT_HASH=$(git rev-parse HEAD)
 
 if [ -z "$PROJECT_NAME" ]; then
   echo "usage: $0 <project_name>"
@@ -31,7 +30,7 @@ fi
 
 cd "$(dirname "$0")"
 
-source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER > /dev/null
+source ./build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 build_local $PROJECT_NAME
 
 if [ -z "$PROJECT_NAME" ]; then

build-system/scripts/build

@@ -58,9 +58,7 @@ function try_fetch_image() {
   return 0
 }
 
-ecr_login
-
-# Ensure ECR repository exists.
+# Login to ECR and ensure repository exists.
 retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle
 
 CONTENT_HASH=$(calculate_content_hash $REPOSITORY)
@@ -78,9 +76,9 @@ fi
 echo "Initializing submodules..."
 init_submodules $REPOSITORY
 
-echo "Checking for terraform..."
 # Validate any terraform if it exists.
 if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then
+  echo "Checking terraform..."
   ensure_terraform
   export TF_IN_AUTOMATION=1
   pushd $ROOT_PATH/$PROJECT_DIR/terraform
@@ -95,18 +93,8 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then
   popd
 fi
 
-# Pull latest parents that are not ours. We also do not want to pull images suffixed by _, this is how we scope intermediate build images.
-PARENTS=$(cat $DOCKERFILE | sed -n -e 's/^FROM \([^[:space:]]\+\).*/\1/p' | sed '/_$/d' | { grep -v $ECR_DEPLOY_URL || true; } | sort | uniq)
-for PARENT in $PARENTS; do
-  [ "$PARENT" == "scratch" ] && continue
-  fetch_image $PARENT
-done
-
-# For each parent that's ours, pull in the latest image.
-PARENTS=$(cat $DOCKERFILE | sed -n -e "s/^FROM $ECR_DEPLOY_URL\/\([^[:space:]]\+\).*/\1/p")
-for PARENT in $PARENTS; do
-  # Extract repository name (i.e. discard tag).
-  PARENT_REPO=${PARENT%:*}
+# For each dependency, pull in the latest image and give it correct tag.
+for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do
   PARENT_CONTENT_HASH=$(calculate_content_hash $PARENT_REPO)
   # There must be a parent image to continue.
   if [ -z "$PARENT_CONTENT_HASH" ]; then
@@ -117,56 +105,19 @@ for PARENT in $PARENTS; do
   echo "Pulling dependency $PARENT_REPO..."
   fetch_image $PARENT_IMAGE_URI
   # Tag it to look like an official release as that's what we use in Dockerfiles.
-  TAG=$ECR_DEPLOY_URL/$PARENT
+  TAG=$ECR_DEPLOY_URL/$PARENT_REPO
   retry docker tag $PARENT_IMAGE_URI $TAG
 done
 
-echo "Extracting commit tag version..."
 COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false)
-
-# Pull, build and push each named stage to cache.
-STAGE_CACHE_FROM=""
-CACHE_FROM=""
-STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p')
-for STAGE in $STAGES; do
-  # Get the last build of this stage to leverage layer caching.
-  if [ -n "$CONTENT_HASH" ]; then
-    echo "Pulling stage: $STAGE"
-    STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$STAGE
-    if try_fetch_image $STAGE_IMAGE_LAST_URI; then
-      STAGE_CACHE_FROM="--cache-from $STAGE_IMAGE_LAST_URI"
-    fi
-  fi
-
-  echo "Building stage: $STAGE"
-  STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$STAGE
-  # Build our dockerfile, add timing information
-  docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH .
-
-  # We don't want to have redo this stages work when building the final image. Use it as a layer cache.
-  CACHE_FROM="--cache-from $STAGE_IMAGE_COMMIT_URI $CACHE_FROM"
-
-  echo "Pushing stage: $STAGE"
-  retry docker push $STAGE_IMAGE_COMMIT_URI > /dev/null 2>&1
-  echo
-done
-
-# Pull previous image to use it as a layer cache if it exists, and if we have not manually ran.
-if [ -n "$COMMIT_HASH" ]; then
-  LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH
-  echo "Pulling previous build of $REPOSITORY..."
-  if try_fetch_image $LAST_SUCCESSFUL_URI ; then
-    CACHE_FROM="--cache-from $LAST_SUCCESSFUL_URI $CACHE_FROM"
-  fi
-  echo
-fi
+echo "Commit tag version: $COMMIT_TAG_VERSION"
 
 # Build the actual image and give it a commit tag.
 IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH
 if [[ -n "$ARCH" ]]; then
   IMAGE_COMMIT_URI=$IMAGE_COMMIT_URI-$ARCH
 fi
 echo "Building image: $IMAGE_COMMIT_URI"
-docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH .
+docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH .
 echo "Pushing image: $IMAGE_COMMIT_URI"
 retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1

build-system/scripts/build_local

@@ -29,7 +29,7 @@ fi
 if [ -n "$ONLY_TARGET" ]; then
   DEPS=("$TARGET_PROJECT")
 else
-  DEPS=($(query_manifest dependencies $TARGET_PROJECT))
+  DEPS=($(query_manifest allDependencies $TARGET_PROJECT))
 fi
 declare -a PROJECTS
 for DEP in ${DEPS[@]}; do
@@ -63,15 +63,30 @@ for E in "${PROJECTS[@]}"; do
     fi
   fi
 
+  IMAGE_URI=$(calculate_image_uri $REPO)
+  ARR=(${IMAGE_URI//:/ })
+  REPO=${ARR[0]}
+  TAG=${ARR[1]}
+
   echo
   echo
   echo
-  echo "*** Building $PROJECT_DIR_NAME:$DOCKERFILE -> $REPO ***"
+  echo "*** Repository: $REPO"
+  echo "*** Tag: $TAG"
+  echo "*** Dockerfile: $DOCKERFILE"
   echo "*** Context: $PWD"
   echo
 
-  docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $ECR_DEPLOY_URL/$REPO:latest .
-  docker tag $ECR_DEPLOY_URL/$REPO:latest aztecprotocol/$REPO:latest
+  if docker image ls --format "{{.Repository}}:{{.Tag}}" | grep -q -w $IMAGE_URI; then
+    echo "Image exists locally. Skipping."
+  else
+    if ecr_login && image_exists $REPO $TAG ; then
+      docker pull $IMAGE_URI
+    else
+      docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $IMAGE_URI .
+      docker tag $IMAGE_URI aztecprotocol/$REPO:latest
+    fi
+  fi
 
   if [ "$PROJECT_DIR_NAME" = "$TARGET_PROJECT" ]; then
     if [ -n "$LAUNCH" ]; then

build-system/scripts/cond_spot_run_container

@@ -0,0 +1,10 @@
+#!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+REPOSITORY=$1
+CPUS=$2
+shift 2
+
+export TAG_POSTFIX=$JOB_NAME
+cond_spot_run_script $REPOSITORY $CPUS cond_run_container $REPOSITORY $@

build-system/scripts/query_manifest

@@ -73,6 +73,9 @@ case "$CMD" in
     yq -r ".\"$REPO\".projectDir // .\"$REPO\".buildDir" $MANIFEST
     ;;
   dependencies)
+    yq -r ".\"$REPO\".dependencies // [] | .[]" $MANIFEST
+    ;;
+  allDependencies)
     # Get dependencies for a given repo. Inclusive of repo itself.
     # If a string, attempt to execute <projectDir>/<string> if exists to get dependencies, else error.
     # If an array, the array lists the dependencies (excluding self).
@@ -102,7 +105,7 @@ case "$CMD" in
     # If no rebuild patterns are given, the result is ["^<projectDir>/"].
     # If a projects rebuildPattern is a string, the rebuild patterns are in <projectDir>/<string>.
     # If an array, the array lists the rebuild patterns.
-    DEPS=($($0 dependencies $REPO))
+    DEPS=($($0 allDependencies $REPO))
     PATTERNS=()
     for DEP in "${DEPS[@]}"; do
       add_rebuild_patterns $DEP

build-system/scripts/setup_env

@@ -81,7 +81,7 @@ echo export ECR_DEPLOY_URL=278380418400.dkr.ecr.eu-west-2.amazonaws.com >> $BASH
 echo export PROJECT=$PROJECT >> $BASH_ENV
 echo export COMMIT_HASH=$COMMIT_HASH >> $BASH_ENV
 echo export COMMIT_TAG=$COMMIT_TAG >> $BASH_ENV
-echo "export COMMIT_MESSAGE='$COMMIT_MESSAGE'" >> $BASH_ENV
+echo "export COMMIT_MESSAGE='${COMMIT_MESSAGE//\'/\'\\\'\'}'" >> $BASH_ENV
 echo export JOB_NAME=$JOB_NAME >> $BASH_ENV
 echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV
 echo export VERSION_TAG=$VERSION_TAG >> $BASH_ENV

build_manifest.yml

@@ -75,16 +75,6 @@ circuits-x86_64-linux-clang-assert:
   dependencies:
     - barretenberg-x86_64-linux-clang
 
-docs:
-  buildDir: .
-  dockerfile: docs/Dockerfile
-  rebuildPatterns:
-    - ^docs/
-    - ^.*.cpp$
-    - ^.*.ts$
-    - ^.release-please-manifest.json$
-    - ^.*/noir-version.json$
-
 l1-contracts:
   buildDir: l1-contracts
 
@@ -98,8 +88,6 @@ noir-contracts-build:
   rebuildPatterns:
     - ^yarn-project/noir-contracts/
     - ^yarn-project/aztec-nr/
-  # Prevent automatic querying of get_dependencies.sh script. Needed because this looks like a yarn project.
-  dependencies: []
 
 yarn-project-base:
   buildDir: yarn-project
@@ -113,6 +101,9 @@ yarn-project-base:
 
 yarn-project:
   buildDir: yarn-project
+  rebuildPatterns:
+    - ^yarn-project/.*\\.(ts|js|cjs|mjs|json|html)$
+    - ^yarn-project/Dockerfile
   dependencies:
     - yarn-project-base
     - noir-contracts-build
@@ -166,3 +157,15 @@ end-to-end:
   projectDir: yarn-project/end-to-end
   dependencies:
     - yarn-project
+
+docs:
+  buildDir: .
+  dockerfile: docs/Dockerfile
+  rebuildPatterns:
+    - ^docs/
+    - ^.*.cpp$
+    - ^.*.ts$
+    - ^.release-please-manifest.json$
+    - ^.*/noir-version.json$
+  dependencies:
+    - yarn-project

docs/Dockerfile

@@ -1,7 +1,5 @@
-FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project
 WORKDIR /usr/src
 COPY . .
-WORKDIR /usr/src/yarn-project
-RUN yarn build
 WORKDIR /usr/src/docs
 RUN yarn && yarn build

docs/Dockerfile.dockerignore

@@ -0,0 +1,10 @@
+# The build context for docs is the root of the repository.
+# Be very specific about what we include.
+*
+
+!docs
+!l1-contracts/src
+!l1-contracts/test
+!barretenberg/cpp/src/barretenberg
+!circuits/cpp/src
+!.release-please-manifest.json

yarn-project/.dockerignore

@@ -10,6 +10,8 @@
 **/*.tsbuildinfo
 **/Dockerfile*
 **/node_modules
+Dockerfile
 
 noir-contracts/src/types
-noir-contracts/src/artifacts
+noir-contracts/src/artifacts
+noir-contracts/target