fix: add retry to tag and docker actions (#2099)

We seem to get throttled around actions to do with our images and tags.

.circleci/config.yml

@@ -32,18 +32,18 @@ checkout: &checkout
   run:
     name: "Checkout code"
     command: |
-      function retry_10() {
-        # Retries up to 10 times with 10 second intervals
-        for i in $(seq 1 10); do
+      function retry() {
+        # Retries up to 3 times with 10 second intervals
+        for i in $(seq 1 3); do
           "$@" && return || sleep 10
         done
-        echo "$@ failed after 10 attempts"
+        echo "$@ failed after 3 attempts"
         exit 1
       }
       cd $HOME
       mkdir -p .ssh
       chmod 0700 .ssh
-      retry_10 ssh-keyscan -t rsa github.com >> .ssh/known_hosts
+      retry ssh-keyscan -t rsa github.com >> .ssh/known_hosts
 
       # A read only key for cloning the repository.
       echo $GIT_CHECKOUT_KEY | base64 -d > .ssh/id_rsa
@@ -57,7 +57,7 @@ checkout: &checkout
       git remote add origin $CIRCLE_REPOSITORY_URL
 
       # Only download metadata when fetching.
-      retry_10 git fetch --depth 1 --filter=blob:none origin $CIRCLE_SHA1
+      retry git fetch --depth 1 --filter=blob:none origin $CIRCLE_SHA1
       git checkout FETCH_HEAD
 
 # Called setup_env to setup a bunch of global variables used throughout the rest of the build process.

build-system/scripts/build

@@ -35,7 +35,18 @@ echo "Working directory: $PWD"
 echo "Dockerfile: $DOCKERFILE"
 echo "Build directory: $BUILD_DIR"
 
+# Fetch images with retries
 function fetch_image() {
+  echo "Pulling: $1"
+  if ! retry docker pull $1 > /dev/null 2>&1; then
+    echo "Image not found: $1"
+    return 1
+  fi
+  return 0
+}
+
+# Fetch images, but don't assume this will work
+function try_fetch_image() {
   echo "Pulling: $1"
   if ! docker pull $1 > /dev/null 2>&1; then
     echo "Image not found: $1"
@@ -44,8 +55,10 @@ function fetch_image() {
   return 0
 }
 
+echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin
+
 # Ensure ECR repository exists.
-retry_10 ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle
+retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle
 
 CONTENT_HASH=$(calculate_content_hash $REPOSITORY)
 echo "Content hash: $CONTENT_HASH"
@@ -78,7 +91,6 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then
 fi
 
 # Pull latest parents that are not ours. We also do not want to pull images suffixed by _, this is how we scope intermediate build images.
-echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin
 PARENTS=$(cat $DOCKERFILE | sed -n -e 's/^FROM \([^[:space:]]\+\).*/\1/p' | sed '/_$/d' | grep -v $ECR_DEPLOY_URL  | sort | uniq)
 for PARENT in $PARENTS; do
   fetch_image $PARENT
@@ -99,7 +111,7 @@ for PARENT in $PARENTS; do
   echo "Pulling dependency $PARENT_REPO..."
   fetch_image $PARENT_IMAGE_URI
   # Tag it to look like an official release as that's what we use in Dockerfiles.
-  docker tag $PARENT_IMAGE_URI $ECR_DEPLOY_URL/$PARENT
+  retry docker tag $PARENT_IMAGE_URI $ECR_DEPLOY_URL/$PARENT
 done
 
 
@@ -159,15 +171,15 @@ for STAGE in $STAGES; do
   CACHE_FROM="--cache-from $STAGE_IMAGE_COMMIT_URI $CACHE_FROM"
 
   echo "Pushing stage: $STAGE"
-  docker push $STAGE_IMAGE_COMMIT_URI > /dev/null 2>&1
+  retry docker push $STAGE_IMAGE_COMMIT_URI > /dev/null 2>&1
   echo
 done
 
 # Pull previous image to use it as a layer cache if it exists.
 if [ -n "$CONTENT_HASH" ]; then
   LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH
   echo "Pulling previous build of $REPOSITORY..."
-  fetch_image $LAST_SUCCESSFUL_URI || true
+  try_fetch_image $LAST_SUCCESSFUL_URI || true
   CACHE_FROM="--cache-from $LAST_SUCCESSFUL_URI $CACHE_FROM"
   echo
 fi
@@ -179,5 +191,5 @@ echo "Building image: $IMAGE_COMMIT_URI"
 docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . \
 | while read line ; do echo "$(date "+%H:%M:%S")| $line"; done
 echo "Pushing image: $IMAGE_COMMIT_URI"
-docker push $IMAGE_COMMIT_URI > /dev/null 2>&1
+retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1
 untag_remote_image $REPOSITORY tainted

build-system/scripts/build_local

@@ -50,7 +50,7 @@ for E in ${PROJECTS[@]}; do
   echo
 
   time docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $ECR_DEPLOY_URL/$REPO:latest .
-  docker tag $ECR_DEPLOY_URL/$REPO:latest aztecprotocol/$REPO:latest
+  retry docker tag $ECR_DEPLOY_URL/$REPO:latest aztecprotocol/$REPO:latest
 
   if [ -n "$LAUNCH" ]; then
     docker run -ti --rm aztecprotocol/$REPO:latest

build-system/scripts/cond_spot_run_build

@@ -16,5 +16,5 @@ cd $(query_manifest buildDir $REPOSITORY)
 if ! check_rebuild cache-$CONTENT_HASH $REPOSITORY; then
   init_submodules $REPOSITORY
   spot_run_script $CONTENT_HASH $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@
-  tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH
+  retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH
 fi

build-system/scripts/cond_spot_run_script

@@ -27,5 +27,5 @@ echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG"
 if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then
   init_submodules $REPOSITORY
   spot_run_script $CONTENT_HASH $@
-  tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG
+  retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG
 fi

build-system/scripts/deploy

@@ -26,4 +26,4 @@ for SERVICE in $SERVICES; do
 done
 
 # Tag the image as deployed.
-tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed
+retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed

build-system/scripts/deploy_dockerhub

@@ -36,20 +36,20 @@ fi
 echo "Deploying to dockerhub: $IMAGE_DEPLOY_URI"
 
 # Login.
-retry_10 ensure_repo $REPOSITORY $ECR_DEPLOY_REGION
-
-# Pull image.
-docker pull $IMAGE_COMMIT_URI
+retry ensure_repo $REPOSITORY $ECR_DEPLOY_REGION
 
 # Login to dockerhub.
 echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin
 
+# Pull image.
+retry docker pull $IMAGE_COMMIT_URI
+
 # Tag with commit tag
-docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_URI
+retry docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_URI
 # Tag with :latest
-docker tag $IMAGE_COMMIT_URI $IMAGE_LATEST_URI
+retry docker tag $IMAGE_COMMIT_URI $IMAGE_LATEST_URI
 
 # Push tagged image to dockerhub.
-docker push $IMAGE_DEPLOY_URI
+retry docker push $IMAGE_DEPLOY_URI
 # Push :latest image to dockerhub
-docker push $IMAGE_LATEST_URI
+retry docker push $IMAGE_LATEST_URI

build-system/scripts/deploy_ecr

@@ -5,21 +5,21 @@ REPOSITORY=$1
 IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY)
 
 # Login to build region and pull the build.
-retry_10 ensure_repo $REPOSITORY $ECR_REGION
-docker pull $IMAGE_COMMIT_URI > /dev/null 2>&1
+retry ensure_repo $REPOSITORY $ECR_REGION
+retry docker pull $IMAGE_COMMIT_URI > /dev/null 2>&1
 
 # Ensure ECR repository exists in deployment region.
-retry_10 ensure_repo $REPOSITORY $ECR_DEPLOY_REGION
+retry ensure_repo $REPOSITORY $ECR_DEPLOY_REGION
 
 # Push image to deployment repo with commit hash tag e.g:
 # falafel:deadbeefcafebabe1337c0de
 IMAGE_DEPLOY_COMMIT_URI=$ECR_DEPLOY_URL/$REPOSITORY:$COMMIT_HASH
-docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_COMMIT_URI
-docker push $IMAGE_DEPLOY_COMMIT_URI
+retry docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_COMMIT_URI
+retry docker push $IMAGE_DEPLOY_COMMIT_URI
 
 # Tag image with full version if we have one. Allows deployment of precise image version if rollback needed.
 if [ -n "${COMMIT_TAG:-}" ]; then
-  tag_remote_image $REPOSITORY $COMMIT_HASH $PROJECT-$COMMIT_TAG $ECR_DEPLOY_REGION
+  retry tag_remote_image $REPOSITORY $COMMIT_HASH $PROJECT-$COMMIT_TAG $ECR_DEPLOY_REGION
 fi
 
-tag_remote_image $REPOSITORY $COMMIT_HASH $DEPLOY_TAG $ECR_DEPLOY_REGION
+retry tag_remote_image $REPOSITORY $COMMIT_HASH $DEPLOY_TAG $ECR_DEPLOY_REGION

build-system/scripts/deploy_global

@@ -22,4 +22,4 @@ deploy_terraform "" ./terraform
 deploy_service $REPOSITORY
 
 # Tag the image as deployed.
-tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed
+retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed

build-system/scripts/extract_repo

@@ -8,7 +8,7 @@ EXTRACT_TO=${3:-./}
 IMAGE_COMMIT_URI=$ECR_DEPLOY_URL/$REPOSITORY:$COMMIT_HASH
 
 echo "Pulling $IMAGE_COMMIT_URI..."
-docker pull $IMAGE_COMMIT_URI > /dev/null 2>&1
+retry docker pull $IMAGE_COMMIT_URI > /dev/null 2>&1
 TEMP_CONTAINER=$(docker create $IMAGE_COMMIT_URI)
 
 echo "Extracting $EXTRACT_FROM from $REPOSITORY to $EXTRACT_TO..."

build-system/scripts/init_submodules

@@ -8,12 +8,12 @@ BUILD_DIR=$(query_manifest buildDir $REPOSITORY)
 if [ -d "$BUILD_DIR" ] && [ "$(git submodule status $BUILD_DIR)" ]; then
   # We have submodules, initialize them
   echo "Initialising any submodules under $REPOSITORY buildDir: $BUILD_DIR"
-  retry_10 git submodule update --init --recursive $BUILD_DIR && exit
+  retry git submodule update --init --recursive $BUILD_DIR && exit
 fi
 
 SUBMODULE_PATH=$(query_manifest submodulePath $REPOSITORY)
 if [ -n "$SUBMODULE_PATH" ]; then
   # TODO: Needs to actually init all dependency submodules as well.
   echo "Initialising submodule: $SUBMODULE_PATH"
-  retry_10 git submodule update --init --recursive $SUBMODULE_PATH
+  retry git submodule update --init --recursive $SUBMODULE_PATH
 fi

build-system/scripts/retry

@@ -0,0 +1,8 @@
+set -eu
+ATTEMPTS=3
+# Retries up to 3 times with 10 second intervals
+for i in $(seq 1 $ATTEMPTS); do
+    "$@" && exit || sleep 10
+done
+echo "$@ failed after $ATTEMPTS attempts"
+exit 1

circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh

@@ -4,10 +4,11 @@
 set -eu
 
 $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null
+export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts"
 REPOSITORY=barretenberg-x86_64-linux-clang-assert
 # use the image rebuild patterns to compute a content hash, use this to get a URI
-IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY)
-docker pull $IMAGE_URI
+IMAGE_URI=$(calculate_image_uri $REPOSITORY)
+retry docker pull $IMAGE_URI
 
 TESTS=(
   crypto_aes128_tests

circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests

@@ -19,9 +19,10 @@ shift # to aztec's circuits `run_tests_local`
 
 $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null
 
+export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts"
 REPOSITORY="barretenberg-circuits-${ARCH}-linux-clang-builder-runner"
-IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY)
-docker pull $IMAGE_URI
+IMAGE_URI=$(calculate_image_uri $REPOSITORY)
+retry docker pull $IMAGE_URI
 
 if [ "$ARCH" != "wasm" ]; then
   # x86_64 / anything other than wasm

circuits/cpp/barretenberg/cpp/scripts/run_tests

@@ -14,8 +14,9 @@ $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null
 
 # use the image rebuild patterns to compute a content hash, use this to get a URI
 REPOSITORY=barretenberg-x86_64-linux-clang-assert
-IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY)
-docker pull $IMAGE_URI
+export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts"
+IMAGE_URI=$(calculate_image_uri $REPOSITORY)
+retry docker pull $IMAGE_URI
 
 if [ -f "$TESTS" ]; then
   TESTS=$(cat $TESTS | tr '\n' ' ')

circuits/cpp/barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -174,7 +174,7 @@ void writeVk(const std::string& bytecodePath, const std::string& outputPath)
  * - Filesystem: The Solidity verifier contract is written to the path specified by outputPath
  *
  * Note: The fact that the contract was computed is for an ACIR circuit is not of importance
- * because this method uses the verification key to compute the Solidity verifier contract.
+ * because this method uses the verification key to compute the Solidity verifier contract
  *
  * @param output_path Path to write the contract to
  * @param vk_path Path to the file containing the serialized verification key

circuits/cpp/scripts/run_tests

@@ -27,7 +27,7 @@ $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null
 
 REPOSITORY="circuits-$ARCH-linux-clang-assert"
 IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY)
-docker pull $IMAGE_URI
+$(git rev-parse --show-toplevel)/build-system/scripts/retry docker pull $IMAGE_URI
 
 # run tests in docker image
 RUN_ARGS="$@" # helper var necessary for some reason to pass all args to docker run

yarn-project/canary/scripts/cond_run_script

@@ -30,5 +30,5 @@ echo "Script to run is $SCRIPT_TO_RUN $@"
 
 if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then
   "$SCRIPT_TO_RUN" "$@"
-  tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG
+  retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG
 fi

yarn-project/canary/scripts/run_tests

@@ -14,13 +14,14 @@ fi
 
 aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com
 
-IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $IMAGE)
-docker pull $IMAGE_URI
-docker tag $IMAGE_URI aztecprotocol/canary:latest
+export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts"
+IMAGE_URI=$(calculate_image_uri $IMAGE)
+retry docker pull $IMAGE_URI
+retry docker tag $IMAGE_URI aztecprotocol/canary:latest
 
-SANDBOX_IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri aztec-sandbox)
-docker pull $SANDBOX_IMAGE_URI
-docker tag $SANDBOX_IMAGE_URI aztecprotocol/sandbox:latest
+SANDBOX_IMAGE_URI=$(calculate_image_uri aztec-sandbox)
+retry docker pull $SANDBOX_IMAGE_URI
+retry docker tag $SANDBOX_IMAGE_URI aztecprotocol/sandbox:latest
 
 docker-compose -f $COMPOSE_FILE rm -f
 docker-compose -f $COMPOSE_FILE up --exit-code-from canary

yarn-project/end-to-end/scripts/cond_run_script

@@ -30,5 +30,5 @@ echo "Script to run is $SCRIPT_TO_RUN $@"
 
 if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then
   "$SCRIPT_TO_RUN" "$@"
-  tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG
+  retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG
 fi

yarn-project/end-to-end/scripts/run_tests

@@ -8,11 +8,11 @@ export COMPOSE_FILE=${2:-docker-compose.yml}
 
 $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null
 
-BS=$(git rev-parse --show-toplevel)/build-system/scripts
+export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts"
 
 for REPO in aztec.js end-to-end aztec-sandbox; do
-  docker pull $($BS/calculate_image_uri $REPO)
-  docker tag $($BS/calculate_image_uri $REPO) aztecprotocol/$REPO:latest
+  retry docker pull $($calculate_image_uri $REPO)
+  retry docker tag $($calculate_image_uri $REPO) aztecprotocol/$REPO:latest
 done
 
 docker-compose rm -f

yarn-project/end-to-end/scripts/run_tests_local

@@ -8,11 +8,11 @@ export COMPOSE_FILE=${2:-./scripts/docker-compose.yml}
 
 aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com
 
-BS=$(git rev-parse --show-toplevel)/build-system/scripts
+export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts"
 
 for REPO in aztec.js end-to-end aztec-sandbox; do
-  docker pull $($BS/calculate_image_uri $REPO)
-  docker tag $($BS/calculate_image_uri $REPO) aztecprotocol/$REPO:latest
+  retry docker pull $(calculate_image_uri $REPO)
+  retry docker tag $(calculate_image_uri $REPO) aztecprotocol/$REPO:latest
 done
 
 docker-compose -f $COMPOSE_FILE rm -f