diff --git a/.circleci/config.yml b/.circleci/config.yml index 41e28e5d328..89d875c64f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -572,7 +572,7 @@ jobs: name: "Build and test" command: build aztec-rpc - aztec-sandbox: + aztec-sandbox-base: machine: image: ubuntu-2004:202010-01 resource_class: large @@ -581,7 +581,40 @@ jobs: - *setup_env - run: name: "Build and test" - command: force_deploy_build aztec-sandbox + command: force_deploy_build aztec-sandbox-base false + + aztec-sandbox-x86_64: + machine: + image: ubuntu-2004:202010-01 + resource_class: large + steps: + - *checkout + - *setup_env + - run: + name: "Build and test" + command: force_deploy_build aztec-sandbox false x86_64 + + aztec-sandbox-arm64: + machine: + image: ubuntu-2204:2023.07.1 + resource_class: arm.large + steps: + - *checkout + - *setup_env + - run: + name: "Build and test" + command: force_deploy_build aztec-sandbox false arm64 + + aztec-sandbox-ecr-manifest: + machine: + image: ubuntu-2004:202010-01 + resource_class: large + steps: + - *checkout + - *setup_env + - run: + name: "Create ECR manifest" + command: create_ecr_manifest aztec-sandbox-base aztec-sandbox x86_64,arm64 circuits-js: machine: @@ -1146,6 +1179,7 @@ jobs: command: | deploy_ecr aztec-sandbox deploy_npm aztec-sandbox + deploy-dockerhub: machine: image: ubuntu-2004:202010-01 @@ -1157,8 +1191,11 @@ jobs: name: "deploy-sandbox" working_directory: aztec-sandbox command: | - deploy_ecr aztec-sandbox - deploy_dockerhub aztec-sandbox + deploy_ecr aztec-sandbox x86_64 + deploy_ecr aztec-sandbox arm64 + deploy_dockerhub aztec-sandbox x86_64 + deploy_dockerhub aztec-sandbox arm64 + create_dockerhub_manifest aztec-sandbox x86_64,arm64 deploy-end: docker: @@ -1373,9 +1410,24 @@ workflows: - types: *yarn_project - circuits-js: *yarn_project - rollup-provider: *yarn_project - - aztec-sandbox: *yarn_project + - aztec-sandbox-base: *yarn_project - canary: *yarn_project + - aztec-sandbox-x86_64: + requires: + - aztec-sandbox-base + <<: *defaults + - aztec-sandbox-arm64: + requires: + - aztec-sandbox-base + <<: *defaults + + - aztec-sandbox-ecr-manifest: + requires: + - aztec-sandbox-x86_64 + - aztec-sandbox-arm64 + <<: *defaults + - e2e-join: requires: - aztec-js @@ -1399,7 +1451,7 @@ workflows: - types - circuits-js - rollup-provider - - aztec-sandbox + - aztec-sandbox-ecr-manifest - canary <<: *defaults @@ -1472,7 +1524,7 @@ workflows: - guides-sample-dapp <<: *defaults - # Deployment and Canary tests + # Deployment and Canary tests - deploy-dockerhub: requires: - e2e-end diff --git a/build-system/scripts/build b/build-system/scripts/build index 6b16d320476..1c7ea38f20c 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -26,6 +26,7 @@ set -euo pipefail REPOSITORY=$1 FORCE_BUILD=${2:-"false"} +ARCH=${3:-""} DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) PROJECT_DIR=$(query_manifest projectDir $REPOSITORY) BUILD_DIR=$(query_manifest buildDir $REPOSITORY) @@ -34,6 +35,7 @@ echo "Repository: $REPOSITORY" echo "Working directory: $PWD" echo "Dockerfile: $DOCKERFILE" echo "Build directory: $BUILD_DIR" +echo "Arch: $ARCH" # Fetch images with retries function fetch_image() { @@ -112,7 +114,8 @@ for PARENT in $PARENTS; do echo "Pulling dependency $PARENT_REPO..." fetch_image $PARENT_IMAGE_URI # Tag it to look like an official release as that's what we use in Dockerfiles. - retry docker tag $PARENT_IMAGE_URI $ECR_DEPLOY_URL/$PARENT + TAG=$ECR_DEPLOY_URL/$PARENT + retry docker tag $PARENT_IMAGE_URI $TAG done @@ -183,6 +186,9 @@ fi # Build the actual image and give it a commit tag. IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH +if [[ -n "$ARCH" ]]; then + IMAGE_COMMIT_URI=$IMAGE_COMMIT_URI-$ARCH +fi echo "Building image: $IMAGE_COMMIT_URI" # Build our dockerfile, add timing information docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . \ diff --git a/build-system/scripts/create_dockerhub_manifest b/build-system/scripts/create_dockerhub_manifest new file mode 100755 index 00000000000..d8416a10bf1 --- /dev/null +++ b/build-system/scripts/create_dockerhub_manifest @@ -0,0 +1,82 @@ +#!/bin/bash +# This script: +# 1. Logs into DockerHub +# 2. Creates 2 manifest lists, the first is version tagged, the second is 'latest' tagged +# 3. Adds the arch specific tagged image to each list +# 4. Pushes the 2 lists + +set -eu + +if [ -z "$COMMIT_TAG" ]; then + echo "Will only push tagged builds to dockerhub. Skipping." + exit 0 +fi + +REPOSITORY=$1 +ARCH_LIST=$2 + +echo "Repo: $REPOSITORY" +echo "Arch list: $ARCH_LIST" + +ACCOUNT="aztecprotocol" +USERNAME="aztecprotocolci" + +COMMIT_TAG_VERSION=$COMMIT_TAG # default unless repo-specific +# Check if it's a repo-specific tag +if [[ "$COMMIT_TAG" == *"/"* ]]; then + REPO_NAME="${COMMIT_TAG%%/*}" + COMMIT_TAG_VERSION="${COMMIT_TAG#*/}" + echo "Tag was made for: $REPO_NAME" + echo "Version: $COMMIT_TAG_VERSION" + + # Check if REPO_NAME is equal to REPOSITORY + if [ "$REPO_NAME" != "$REPOSITORY" ]; then + echo "REPO_NAME ($REPO_NAME) does not match REPOSITORY ($REPOSITORY). Exiting..." + exit 1 + fi +fi + +# Check it's a valid semver. +VERSION=$(npx semver $COMMIT_TAG_VERSION) +if [ -z "$VERSION" ]; then + echo "$COMMIT_TAG_VERSION is not a semantic version." + exit 1 +fi + +# We now have the tage for each image +IMAGE_TAG=$COMMIT_TAG_VERSION + +MANIFEST_DEPLOY_URI=$ACCOUNT/$REPOSITORY:$IMAGE_TAG +MANIFEST_LATEST_URI=$ACCOUNT/$REPOSITORY:latest + +# Login to dockerhub. +echo "$DOCKERHUB_PASSWORD" | docker login -u $USERNAME --password-stdin + +export DOCKER_CLI_EXPERIMENTAL=enabled + +OLD_IFS=$IFS +IFS=',' + +# For each arch, add the tagged image to 2 manifest lists. One tagged with the version, the other with 'latest' +for A in $ARCH_LIST +do + IMAGE_DEPLOY_URI=$ACCOUNT/$REPOSITORY:$IMAGE_TAG-$A + echo "Adding image $IMAGE_DEPLOY_URI to manifest list $MANIFEST_DEPLOY_URI" + docker manifest create $MANIFEST_DEPLOY_URI \ + --amend $IMAGE_DEPLOY_URI + + echo "Adding image $IMAGE_DEPLOY_URI to manifest list $MANIFEST_LATEST_URI" + docker manifest create $MANIFEST_LATEST_URI \ + --amend $IMAGE_DEPLOY_URI +done + +IFS=$OLD_IFS +unset OLD_IFS + +echo "Pushing manifest list $MANIFEST_DEPLOY_URI..." +# Push the version tagged list +docker manifest push --purge $MANIFEST_DEPLOY_URI + +echo "Pushing manifest list $MANIFEST_LATEST_URI..." +# Push the latest tagged list +docker manifest push --purge $MANIFEST_LATEST_URI \ No newline at end of file diff --git a/build-system/scripts/create_ecr_manifest b/build-system/scripts/create_ecr_manifest new file mode 100755 index 00000000000..6163c952bb4 --- /dev/null +++ b/build-system/scripts/create_ecr_manifest @@ -0,0 +1,38 @@ +#!/bin/bash +# This script: +# 1. Logs into ECR and ensures we have the given repository +# 2. Computes the image uri of the cached images for the given repository given the list of architectures +# 3. Creates a manifest list using a platform agnositc image uri, adds each image to it +# 4. Pushes the manifest list + +set -e + +REPOSITORY=$1 +FINAL_IMAGE_NAME=$2 +ARCH_LIST=$3 + +# Ensure ECR repository exists. +retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle + +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Content hash: $CONTENT_HASH" + +FINAL=$ECR_URL/$FINAL_IMAGE_NAME:cache-$CONTENT_HASH + +echo "Creating manifest list $FINAL..." + +export DOCKER_CLI_EXPERIMENTAL=enabled + +OLD_IFS=$IFS +IFS=',' +for A in $ARCH_LIST +do + IMAGE=$ECR_URL/$FINAL_IMAGE_NAME:cache-$CONTENT_HASH-$A + echo "Adding image $IMAGE to manifest list" + docker manifest create $FINAL \ + --amend $IMAGE +done +IFS=$OLD_IFS +unset OLD_IFS + +docker manifest push --purge $FINAL \ No newline at end of file diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 99cdd046410..b6d165a12e0 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -7,8 +7,22 @@ if [ -z "$COMMIT_TAG" ]; then fi REPOSITORY=$1 +ARCH=${2:-""} + +echo "Repo: $REPOSITORY" +echo "Arch: $ARCH" + +ACCOUNT="aztecprotocol" +USERNAME="aztecprotocolci" + IMAGE_COMMIT_URI=$ECR_DEPLOY_URL/$REPOSITORY:$COMMIT_HASH -IMAGE_LATEST_URI=aztecprotocol/$REPOSITORY:latest + +IMAGE_LATEST_URI=$ACCOUNT/$REPOSITORY:latest +if [[ -n "$ARCH" ]]; then + IMAGE_COMMIT_URI=$IMAGE_COMMIT_URI-$ARCH + IMAGE_LATEST_URI=$IMAGE_LATEST_URI-$ARCH +fi + COMMIT_TAG_VERSION=$COMMIT_TAG # default unless repo-specific # Check if it's a repo-specific tag if [[ "$COMMIT_TAG" == *"/"* ]]; then @@ -24,7 +38,10 @@ if [[ "$COMMIT_TAG" == *"/"* ]]; then fi fi IMAGE_TAG=$COMMIT_TAG_VERSION -IMAGE_DEPLOY_URI=aztecprotocol/$REPOSITORY:$IMAGE_TAG +IMAGE_DEPLOY_URI=$ACCOUNT/$REPOSITORY:$IMAGE_TAG +if [[ -n "$ARCH" ]]; then + IMAGE_DEPLOY_URI=$IMAGE_DEPLOY_URI-$ARCH +fi # Check it's a valid semver. VERSION=$(npx semver $COMMIT_TAG_VERSION) @@ -39,13 +56,17 @@ echo "Deploying to dockerhub: $IMAGE_DEPLOY_URI" retry ensure_repo $REPOSITORY $ECR_DEPLOY_REGION # Login to dockerhub. -echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin +echo "$DOCKERHUB_PASSWORD" | docker login -u $USERNAME --password-stdin +echo "Pulling $IMAGE_COMMIT_URI" # Pull image. retry docker pull $IMAGE_COMMIT_URI +echo "Tagging $IMAGE_COMMIT_URI as $IMAGE_DEPLOY_URI" # Tag with commit tag retry docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_URI + +echo "Tagging $IMAGE_COMMIT_URI as $IMAGE_LATEST_URI" # Tag with :latest retry docker tag $IMAGE_COMMIT_URI $IMAGE_LATEST_URI diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr index 35be5922eaf..2fb0cd2fbe7 100755 --- a/build-system/scripts/deploy_ecr +++ b/build-system/scripts/deploy_ecr @@ -2,7 +2,11 @@ set -eu REPOSITORY=$1 +ARCH=${2:-""} IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) +if [[ -n "$ARCH" ]]; then + IMAGE_COMMIT_URI=$IMAGE_COMMIT_URI-$ARCH +fi # Login to build region and pull the build. retry ensure_repo $REPOSITORY $ECR_REGION @@ -11,9 +15,12 @@ retry docker pull $IMAGE_COMMIT_URI > /dev/null 2>&1 # Ensure ECR repository exists in deployment region. retry ensure_repo $REPOSITORY $ECR_DEPLOY_REGION -# Push image to deployment repo with commit hash tag e.g: -# falafel:deadbeefcafebabe1337c0de +# Push image to deployment repo with commit hash tag and optional architecture e.g: +# falafel:deadbeefcafebabe1337c0de-x86_64 IMAGE_DEPLOY_COMMIT_URI=$ECR_DEPLOY_URL/$REPOSITORY:$COMMIT_HASH +if [[ -n "$ARCH" ]]; then + IMAGE_DEPLOY_COMMIT_URI=$IMAGE_DEPLOY_COMMIT_URI-$ARCH +fi retry docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_COMMIT_URI retry docker push $IMAGE_DEPLOY_COMMIT_URI diff --git a/build-system/scripts/force_deploy_build b/build-system/scripts/force_deploy_build index d5d9e46e71b..e5a76665b16 100755 --- a/build-system/scripts/force_deploy_build +++ b/build-system/scripts/force_deploy_build @@ -2,16 +2,18 @@ # # This script forces a build if we're in a deployment run, otherwise runs `build` as usual # -# usage: ./deploy_force_build -# example: ./deploy_force_build aztec-sandbox +# usage: ./deploy_force_build +# example: ./deploy_force_build aztec-sandbox true set -e REPOSITORY=$1 -FORCE_BUILD=${2:-"false"} +shift +FORCE_BUILD=$1 +shift # if FORCE_BUILD is already set, just continue with it if [[ $FORCE_BUILD == 'true' ]]; then - build $REPOSITORY true + build $REPOSITORY true $@ exit 0 fi @@ -37,12 +39,12 @@ if [[ -n "${COMMIT_TAG:-}" ]]; then VERSION=$(npx semver $COMMIT_TAG_VERSION) if [ -z "$VERSION" ]; then # Not a version tag, build normally - build $REPOSITORY + build $REPOSITORY false $@ else # Force build - build $REPOSITORY true + build $REPOSITORY true $@ fi else # Not a tagged commit, build normally - build $REPOSITORY + build $REPOSITORY false $@ fi diff --git a/build_manifest.json b/build_manifest.json index 3a24db1ef01..2d55f73f907 100644 --- a/build_manifest.json +++ b/build_manifest.json @@ -112,10 +112,15 @@ "buildDir": "yarn-project", "projectDir": "yarn-project/aztec-rpc" }, - "aztec-sandbox": { + "aztec-sandbox-base": { "buildDir": "yarn-project", "projectDir": "yarn-project/aztec-sandbox" }, + "aztec-sandbox": { + "buildDir": "yarn-project", + "projectDir": "yarn-project/aztec-sandbox", + "dockerfile": "Dockerfile.final" + }, "aztec.js": { "buildDir": "yarn-project", "projectDir": "yarn-project/aztec.js" diff --git a/yarn-project/aztec-sandbox/Dockerfile.final b/yarn-project/aztec-sandbox/Dockerfile.final new file mode 100644 index 00000000000..a9cc84909f1 --- /dev/null +++ b/yarn-project/aztec-sandbox/Dockerfile.final @@ -0,0 +1,10 @@ +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec-sandbox-base AS builder + +FROM node:18-alpine + +COPY --from=builder /usr/src/ /usr/src/ +WORKDIR /usr/src/yarn-project/aztec-sandbox + +ENTRYPOINT ["yarn"] +CMD [ "start" ] +EXPOSE 8080 \ No newline at end of file