diff --git a/.circleci/config.yml b/.circleci/config.yml index c1f0318961a..cfbed9d1d76 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,6 +346,17 @@ jobs: name: Build command: build yarn-project | add_timestamps + yarn-project-prod: + machine: + image: ubuntu-2204:2023.07.2 + resource_class: large + steps: + - *checkout + - *setup_env + - run: + name: Build + command: build yarn-project-prod | add_timestamps + yarn-project-formatting: machine: image: ubuntu-2204:2023.07.2 @@ -368,7 +379,7 @@ jobs: name: Test command: cond_spot_run_container yarn-project 64 test | add_timestamps - aztec-sandbox-x86_64: + aztec-sandbox: machine: image: ubuntu-2204:2023.07.2 resource_class: large @@ -379,22 +390,7 @@ jobs: name: "Build and test" command: build aztec-sandbox - aztec-sandbox-arm64: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: arm.large - steps: - - *checkout - - *setup_env - - run: - name: "Build and test" - # We need to force not to use docker buildkit because for some reason on arm only, it ends up making a call - # out to eu-west2 despite the image being locally tagged, resulting in unauthorized 401. Weird docker bug? - command: | - echo "export DOCKER_BUILDKIT=" > $BASH_ENV - build aztec-sandbox - - cli-x86_64: + cli: machine: image: ubuntu-2204:2023.07.2 resource_class: large @@ -405,21 +401,6 @@ jobs: name: "Build and test" command: build cli - cli-arm64: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: arm.large - steps: - - *checkout - - *setup_env - - run: - name: "Build and test" - # We need to force not to use docker buildkit because for some reason on arm only, it ends up making a call - # out to eu-west2 despite the image being locally tagged, resulting in unauthorized 401. Weird docker bug? - command: | - echo "export DOCKER_BUILDKIT=" > $BASH_ENV - build cli - mainnet-fork: machine: image: ubuntu-2204:2023.07.2 @@ -442,21 +423,6 @@ jobs: name: "Build and test" command: build aztec-faucet | add_timestamps - ecr-manifest: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large - steps: - - *checkout - - *setup_env - - run: - name: "Create ECR manifest" - command: | - create_ecr_manifest aztec-sandbox x86_64,arm64 - create_ecr_manifest cli x86_64,arm64 - create_ecr_manifest aztec-faucet x86_64 - create_ecr_manifest mainnet-fork x86_64 - boxes-blank-react: machine: image: ubuntu-2204:2023.07.2 @@ -938,8 +904,8 @@ jobs: name: "Assemble benchmark summary from uploaded logs" command: ./scripts/ci/assemble_e2e_benchmark.sh - # Deploy jobs. - deploy-mainnet-fork: + # Release jobs. + release-npm: machine: image: ubuntu-2204:2023.07.2 resource_class: medium @@ -947,12 +913,12 @@ jobs: - *checkout - *setup_env - run: - name: "Deploy mainnet fork" + name: "yarn-project" command: | - should_deploy || exit 0 - deploy mainnet-fork + should_release || exit 0 + yarn-project/deploy_npm.sh latest - deploy-contracts: + release-dockerhub: machine: image: ubuntu-2204:2023.07.2 resource_class: medium @@ -960,13 +926,17 @@ jobs: - *checkout - *setup_env - run: - name: "Deploy L1 contracts to mainnet fork" - working_directory: l1-contracts + name: "Release to dockerhub" command: | - should_deploy || exit 0 - ./scripts/ci_deploy_contracts.sh + should_release || exit 0 + deploy_dockerhub noir + deploy_dockerhub aztec-sandbox + deploy_dockerhub cli + deploy_dockerhub aztec-faucet + deploy_dockerhub mainnet-fork - deploy-npm: + # Deploy jobs. + deploy-mainnet-fork: machine: image: ubuntu-2204:2023.07.2 resource_class: medium @@ -974,12 +944,12 @@ jobs: - *checkout - *setup_env - run: - name: "yarn-project" + name: "Deploy mainnet fork" command: | should_deploy || exit 0 - yarn-project/deploy_npm.sh latest + deploy mainnet-fork - deploy-dockerhub: + deploy-contracts: machine: image: ubuntu-2204:2023.07.2 resource_class: medium @@ -987,14 +957,11 @@ jobs: - *checkout - *setup_env - run: - name: "Deploy to dockerhub" + name: "Deploy L1 contracts to mainnet fork" + working_directory: l1-contracts command: | should_deploy || exit 0 - deploy_dockerhub noir x86_64,arm64 - deploy_dockerhub aztec-sandbox x86_64,arm64 - deploy_dockerhub cli x86_64,arm64 - deploy_dockerhub aztec-faucet x86_64 - deploy_dockerhub mainnet-fork x86_64 + ./scripts/ci_deploy_contracts.sh deploy-devnet: machine: @@ -1036,6 +1003,11 @@ defaults_yarn_project: &defaults_yarn_project - yarn-project <<: *defaults +defaults_yarn_project_prod: &defaults_yarn_project_prod + requires: + - yarn-project-prod + <<: *defaults + defaults_deploy: &defaults_deploy requires: - end @@ -1122,42 +1094,37 @@ workflows: requires: - yarn-project-base <<: *defaults + - yarn-project-prod: *defaults_yarn_project - yarn-project-formatting: *defaults_yarn_project - yarn-project-tests: *defaults_yarn_project - end-to-end: *defaults_yarn_project - build-docs: *defaults_yarn_project - - aztec-sandbox-x86_64: *defaults_yarn_project - - aztec-sandbox-arm64: *defaults_yarn_project - - cli-x86_64: *defaults_yarn_project - - cli-arm64: *defaults_yarn_project - - aztec-faucet: *defaults_yarn_project - - ecr-manifest: - requires: - - aztec-sandbox-x86_64 - - aztec-sandbox-arm64 - - cli-x86_64 - - cli-arm64 - <<: *defaults + + # Artifacts + - aztec-sandbox: *defaults_yarn_project_prod + - cli: *defaults_yarn_project_prod + - aztec-faucet: *defaults_yarn_project_prod # Boxes. - boxes-blank-react: requires: - - aztec-sandbox-x86_64 + - aztec-sandbox <<: *defaults - boxes-blank: requires: - - aztec-sandbox-x86_64 + - aztec-sandbox <<: *defaults - boxes-token: requires: - - aztec-sandbox-x86_64 + - aztec-sandbox <<: *defaults # End to end tests. - e2e-join: requires: - end-to-end - - ecr-manifest + - aztec-sandbox + - cli <<: *defaults - e2e-2-pxes: *e2e_test - e2e-deploy-contract: *e2e_test @@ -1240,12 +1207,14 @@ workflows: - bench-process-history <<: *defaults - # Production deployment - - deploy-dockerhub: *defaults_deploy - - deploy-npm: *defaults_deploy + # Production releases. + - release-dockerhub: *defaults_deploy + - release-npm: *defaults_deploy + + # Production deployment. - deploy-mainnet-fork: requires: - - deploy-dockerhub + - release-dockerhub <<: *defaults_deploy - deploy-contracts: requires: @@ -1255,4 +1224,3 @@ workflows: requires: - deploy-contracts <<: *defaults_deploy - diff --git a/build-system/scripts/build b/build-system/scripts/build index 71826e74dbf..6906a275149 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -86,30 +86,59 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then popd fi -# For each dependency, pull in the latest image and give it correct tag. +# For each dependency, substitute references to the dependency in dockerfile, with the relevent built image uri. +# We have to perform a bit of probing to determine which actual image we want to use. +# When we used buildx to create a multiarch image, there will be no images with "-$ARCH" suffixes (normalise this?). +# Also we sometimes build an arm image from an x86 parent, so there won't always be an arm parent, and we fallback. for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do - PARENT_IMAGE_URI=$(calculate_image_uri $PARENT_REPO) - echo "Pulling dependency $PARENT_IMAGE_URI..." - if ! fetch_image $PARENT_IMAGE_URI; then - # This is a *bit* of a hack maybe. Some of our arm images can be built from x86 dependents. - # e.g. node projects are architecture independent. - # This may not hold true if we start introducing npm modules that are backed by native code. - # But for now, to avoid building some projects twice, we can fallback onto x86 variant. - PARENT_IMAGE_URI=$(calculate_image_uri $PARENT_REPO x86_64) - echo "Falling back onto x86 build. Pulling dependency $PARENT_IMAGE_URI..." - fetch_image $PARENT_IMAGE_URI + # We want the parent image tag without any arch suffix. + PARENT_IMAGE_TAG=$(calculate_image_tag $PARENT_REPO "") + + # Attempt to locate multiarch image. + if ! image_exists $PARENT_REPO $PARENT_IMAGE_TAG; then + # Attempt to locate our specific arch image. + PARENT_IMAGE_TAG=$(calculate_image_tag $PARENT_REPO) + if ! image_exists $PARENT_REPO $PARENT_IMAGE_TAG; then + # Finally attempt to locate x86_64 image tag, as sometimes we build arch specific images from x86_64 images. + PARENT_IMAGE_TAG=$(calculate_image_tag $PARENT_REPO x86_64) + if ! image_exists $PARENT_REPO $PARENT_IMAGE_TAG; then + echo "Failed to locate multiarch image, arch specific image, or x86_64 image. Aborting." + exit 1 + fi + fi fi - # Tag it to look like an official release as that's what we use in Dockerfiles. - TAG=$ECR_DEPLOY_URL/$PARENT_REPO - docker tag $PARENT_IMAGE_URI $TAG + + # Substitute references to parent repo, with the relevent built image uri. + DEPLOY_URI=$ECR_DEPLOY_URL/$PARENT_REPO + PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:$PARENT_IMAGE_TAG + awk '{if ($1 == "FROM" && $2 == "'$DEPLOY_URI'") $2 = "'$PARENT_IMAGE_URI'"; print $0}' $DOCKERFILE > _temp && mv _temp $DOCKERFILE done COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) echo "Commit tag version: $COMMIT_TAG_VERSION" -# Build the actual image and give it a commit tag. IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) echo "Building image: $IMAGE_COMMIT_URI" -docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . -echo "Pushing image: $IMAGE_COMMIT_URI" -retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 + +MULTIARCH=$(query_manifest multiarch $REPOSITORY) + +# Build the image. +if [ "$MULTIARCH" == "buildx" ]; then + # We've requested to use buildx. This will build both arch containers on the host machine using virtualization. + # The result is a single image tag that supports multiarch. + # This is the simplest approach for build jobs that are not too intensive. + docker buildx create --name builder --use + docker buildx inspect --bootstrap + docker buildx build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH --platform linux/amd64,linux/arm64 . --push +else + # If multiarch is set to "host", the assumption is that we're doing multiple builds on different machine architectures + # in parallel, and that there is a another job that runs afterwards to combine them into a manifest. + # In this case we need to augment the image tag with the hosts architecture to ensure its uniqueness. + if [ "$MULTIARCH" == "host" ]; then + IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY host) + fi + + docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . + echo "Pushing image: $IMAGE_COMMIT_URI" + retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 +fi diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index b4572cecee0..7ef1d3c5d8e 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -98,7 +98,7 @@ for E in "${PROJECTS[@]}"; do echo -e "${YELLOW}Project or dependency has local modifications! Building...${RESET}" docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $DEPLOY_IMAGE_URI . else - if [ -z "$NO_CACHE" ] && docker image ls --format "{{.Repository}}:{{.Tag}}" | grep -q -w $CACHE_IMAGE_URI; then + if [ -z "$NO_CACHE" ] && docker image ls --format "{{.Repository}}:{{.Tag}}" | grep -q -w "$CACHE_IMAGE_URI$"; then echo -e "${GREEN}Image exists locally. Tagging as $DEPLOY_IMAGE_URI${RESET}" docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI else diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 630595aa371..28ae8ff2e62 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -1,21 +1,8 @@ #!/bin/bash - [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -REPOSITORY=$1 -COMMIT_HASH=${2:-${COMMIT_HASH:-$(git rev-parse HEAD)}} - -# Get list of rebuild patterns, concat them with regex 'or' (|), and double escape \ for awk -v. -AWK_PATTERN=$(query_manifest rebuildPatterns $REPOSITORY | tr '\n' '|' | sed 's/\\/\\\\/g') -# Remove the trailing '|'. -AWK_PATTERN=${AWK_PATTERN%|} - -cd "$(git rev-parse --show-toplevel)" +set -euo pipefail -# an example line is +# An example line is: # 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json -# this format is beneficial as it grabs the hashes from git efficiently -# we will next filter by our rebuild patterns -# then we pipe the hash portion of each file to git hash-object to produce our content hash -git ls-tree -r $COMMIT_HASH | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin \ No newline at end of file +# Extract the hashes and pipe the hash portion of each file to git hash-object to produce our content hash. +calculate_rebuild_files "$@" | awk '{print $3}' | git hash-object --stdin \ No newline at end of file diff --git a/build-system/scripts/calculate_image_tag b/build-system/scripts/calculate_image_tag index 2a7786ecdc1..c273648287c 100755 --- a/build-system/scripts/calculate_image_tag +++ b/build-system/scripts/calculate_image_tag @@ -1,9 +1,23 @@ #!/bin/bash +# Return a repositories build cache image tag based on content hash. +# If the second argument is set: +# It's used to suffix the tag with the given unique arch descriptor. +# Unless it's "host" in which case use the host machines arch as the arch descriptor. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu REPOSITORY=$1 -ARCH=${2:-$(uname -m)} -[ "$ARCH" == "aarch64" ] && ARCH=arm64 +ARCH=${2:-} + CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "cache-$CONTENT_HASH-$ARCH" + +if [ -z "$ARCH" ]; then + echo "cache-$CONTENT_HASH" +else + if [ "$ARCH" == "host" ]; then + ARCH=$(uname -m) + fi + # Normalise. + [ "$ARCH" == "aarch64" ] && ARCH=arm64 + echo "cache-$CONTENT_HASH-$ARCH" +fi diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index fe60a9c05ab..8efd7ab4cdc 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -3,7 +3,6 @@ set -eu REPOSITORY=$1 -ARCH=${2:-$(uname -m)} -[ "$ARCH" == "aarch64" ] && ARCH=arm64 -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$ARCH" + +TAG=$(calculate_image_tag "$@") +echo "$ECR_URL/$REPOSITORY:$TAG" diff --git a/build-system/scripts/calculate_rebuild_files b/build-system/scripts/calculate_rebuild_files new file mode 100755 index 00000000000..9be1afd4912 --- /dev/null +++ b/build-system/scripts/calculate_rebuild_files @@ -0,0 +1,20 @@ +#!/bin/bash + +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +COMMIT_HASH=${2:-${COMMIT_HASH:-$(git rev-parse HEAD)}} + +# Get list of rebuild patterns, concat them with regex 'or' (|), and double escape \ for awk -v. +AWK_PATTERN=$(query_manifest rebuildPatterns $REPOSITORY | tr '\n' '|' | sed 's/\\/\\\\/g') +# Remove the trailing '|'. +AWK_PATTERN=${AWK_PATTERN%|} + +cd "$(git rev-parse --show-toplevel)" + +# An example line is: +# 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json +# This format is beneficial as it grabs the hashes from git efficiently. +# We then filter by our rebuild patterns. +git ls-tree -r $COMMIT_HASH | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $0}' \ No newline at end of file diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 6e970387cce..3333e5dec7c 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -6,4 +6,4 @@ REPOSITORY=$1 CPUS=$2 ARCH=${3:-x86_64} -cond_spot_run_script $REPOSITORY $CPUS $ARCH build $REPOSITORY $ARCH | add_timestamps +cond_spot_run_script $REPOSITORY $CPUS $ARCH build $REPOSITORY | add_timestamps diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 89d41c9af0e..4b1832127ac 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -19,8 +19,7 @@ CPUS=$2 ARCH=$3 shift 3 -# If the CPUS have a specific architecture assigned, we need to use that to build the success tag. -BASE_TAG=$(calculate_image_tag $REPOSITORY $ARCH) +BASE_TAG=$(calculate_image_tag $REPOSITORY) SUCCESS_TAG=$BASE_TAG if [ -n "${TAG_POSTFIX:-}" ]; then diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index ac99bbe4f1a..606e0952b5d 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -3,66 +3,48 @@ set -eu REPOSITORY=$1 -# TODO: Why even provide this? We can just figure it out by probing. -ARCH_LIST=${2:-} -# The tag to use for release images. Can be turned into an argument if needed, for now we only release 'latest'. -DIST_TAG="latest" - -function docker_or_dryrun { +function cmd_or_dryrun { if [ "$DRY_DEPLOY" -eq 1 ]; then - echo DRY RUN: docker $@ + echo DRY RUN: $@ else - retry docker $@ + retry $@ fi } echo "Repo: $REPOSITORY" -echo "Arch List: $ARCH_LIST" - -VERSION_TAG=$(extract_tag_version $REPOSITORY false) -# if no version tag, check if we're on `master` branch -if [[ -z "$VERSION_TAG" ]]; then - if [[ "$BRANCH" != "master" ]]; then - echo "No version tag found. Exiting" >&2 - exit 1 - fi - # if we're on master, use the DEPLOY_TAG as the version tag - VERSION_TAG=$DEPLOY_TAG +# Set tag to the commit tag (a version number) if we have one, otherwise branch name normalised to a tag format. +if [ -n "$COMMIT_TAG" ]; then + TAG=$(extract_tag_version $REPOSITORY true) +else + TAG=$(echo "$BRANCH" | sed 's/[^a-zA-Z0-9_.-]/_/g') fi -MANIFEST_DEPLOY_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$VERSION_TAG -MANIFEST_DIST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$DIST_TAG - # Login to dockerhub and ecr dockerhub_login ecr_login -IFS=',' -for ARCH in $ARCH_LIST; do - IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY $ARCH) - retry docker pull $IMAGE_COMMIT_URI - - # Retag and push image. - IMAGE_DEPLOY_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$VERSION_TAG-$ARCH - docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_URI - docker_or_dryrun push $IMAGE_DEPLOY_URI - - echo "Adding image $IMAGE_DEPLOY_URI to manifest list $MANIFEST_DEPLOY_URI..." - docker_or_dryrun manifest create $MANIFEST_DEPLOY_URI --amend $IMAGE_DEPLOY_URI - - # Add latest manifest if we're making a release. - if [[ "$VERSION_TAG" != $DEPLOY_TAG ]]; then - echo "Adding image $IMAGE_DEPLOY_URI to manifest list $MANIFEST_DIST_URI" - docker_or_dryrun manifest create $MANIFEST_DIST_URI --amend $IMAGE_DEPLOY_URI - fi -done - -docker_or_dryrun manifest push --purge $MANIFEST_DEPLOY_URI +# Install skopeo, and immediately hack it to newer version. +sudo apt install -y skopeo +retry wget https://github.com/lework/skopeo-binary/releases/download/v1.13.3/skopeo-linux-amd64 +chmod +x ./skopeo-linux-amd64 && sudo mv ./skopeo-linux-amd64 /usr/bin/skopeo + +# Publish with version or branch tag. +SRC_URI=$(calculate_image_uri $REPOSITORY) +DST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$TAG +cmd_or_dryrun skopeo copy --all docker://$SRC_URI docker://$DST_URI + +# If this was a versioned release, publish as latest. +if [ -n "$COMMIT_TAG" ]; then + SRC_URI=$DST_URI + DST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:latest + cmd_or_dryrun skopeo copy --all docker://$SRC_URI docker://$DST_URI +fi -# Retag version as latest if we're making a release. -if [[ "$VERSION_TAG" != "$DEPLOY_TAG" ]]; then - echo "Tagging $MANIFEST_DEPLOY_URI as $DIST_TAG..." - docker_or_dryrun manifest push --purge $MANIFEST_DIST_URI +# If this was a deployment, publish with the deploy tag. +if [ -n "$DEPLOY_TAG" ]; then + SRC_URI=$DST_URI + DST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$DEPLOY_TAG + cmd_or_dryrun skopeo copy --all docker://$SRC_URI docker://$DST_URI fi diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index 0358fbf89cb..cd0b5c0888b 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -126,4 +126,7 @@ case "$CMD" in if [ "$(git ls-tree HEAD $DIR | awk '{print $2}')" = "commit" ]; then git ls-tree HEAD $DIR | awk '{print $4}' fi + ;; + multiarch) + yq -r ".\"$REPO\".multiarch // false" $MANIFEST esac diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 5bd564818c7..6a166d97236 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -19,6 +19,9 @@ BASH_ENV=${BASH_ENV:-} BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) PROJECT=$(cat PROJECT) COMMIT_MESSAGE=$(git log -n 1 --pretty=format:"%s" $COMMIT_HASH) +PATH=$PATH:$BUILD_SYSTEM_PATH/scripts + +export BRANCH echo "COMMIT_HASH=$COMMIT_HASH" echo "COMMIT_TAG=$COMMIT_TAG" @@ -33,25 +36,29 @@ if [[ "$COMMIT_MESSAGE" == *"[ci dry-deploy]"* ]]; then DRY_DEPLOY=1 fi -if [ -n "${COMMIT_TAG:-}" ]; then - # Extract the deploy env from the commit tag, if it has one, e.g. testnet. - # If we have one, we look something like v2.1.123-testnet.0. This is a "non production" release. - if [[ "$COMMIT_TAG" == *"-"* ]]; then - # Strips the trailing '.XX' from the end of the commit tag - TEMP=${COMMIT_TAG%.*} - # Strips the 'vX.Y.ZZZ-' from the front of the commit tag, leaving the e.g. 'testnet' - DEPLOY_ENV=${TEMP##*-} +if should_deploy; then + if [ -n "${COMMIT_TAG:-}" ]; then + # Extract the deploy env from the commit tag, if it has one, e.g. testnet. + # If we have one, we look something like v2.1.123-testnet.0. This is a "non production" release. + if [[ "$COMMIT_TAG" == *"-"* ]]; then + # Strips the trailing '.XX' from the end of the commit tag + TEMP=${COMMIT_TAG%.*} + # Strips the 'vX.Y.ZZZ-' from the front of the commit tag, leaving the e.g. 'testnet' + DEPLOY_ENV=${TEMP##*-} + else + DEPLOY_ENV=prod + fi else - DEPLOY_ENV=prod + # If we're on master, this is our devnet. + DEPLOY_ENV="dev" fi -else - # We have no commit tag, this is our devnet. - DEPLOY_ENV="dev" fi # - The deploy tag (used in api paths, subdomains), is a concatenation of the project name and deploy environment, # e.g. aztec-dev, aztec-testnet, or aztec-prod -DEPLOY_TAG=$PROJECT-$DEPLOY_ENV +if [ -n "${DEPLOY_ENV:-}" ]; then + DEPLOY_TAG=$PROJECT-$DEPLOY_ENV +fi if [ -z "$BASH_ENV" ]; then BASH_ENV=$(mktemp) @@ -77,8 +84,8 @@ echo export COMMIT_TAG=$COMMIT_TAG >> $BASH_ENV echo "export COMMIT_MESSAGE='${COMMIT_MESSAGE//\'/\'\\\'\'}'" >> $BASH_ENV echo export JOB_NAME=$JOB_NAME >> $BASH_ENV echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV -echo export DEPLOY_ENV=$DEPLOY_ENV >> $BASH_ENV -echo export DEPLOY_TAG=$DEPLOY_TAG >> $BASH_ENV +echo export DEPLOY_ENV=${DEPLOY_ENV:-} >> $BASH_ENV +echo export DEPLOY_TAG=${DEPLOY_TAG:-} >> $BASH_ENV echo export BRANCH=$BRANCH >> $BASH_ENV echo export PULL_REQUEST=$PULL_REQUEST >> $BASH_ENV echo export DRY_DEPLOY=${DRY_DEPLOY:-0} >> $BASH_ENV diff --git a/build-system/scripts/should_deploy b/build-system/scripts/should_deploy index b5f198af06e..53f295b191f 100755 --- a/build-system/scripts/should_deploy +++ b/build-system/scripts/should_deploy @@ -1,14 +1,9 @@ #!/bin/bash # Returns success if we are expected to do a deployment. -# This is if we have a commit tag (release) or if we're on `master` branch (devnet deployment). - +# Right now, that's only if we're master. set -eu -# This is when we only want to deploy on master, not release with new COMMIT_TAG. -# Specifically for deploying devnet. -RELEASE=${1:-1} - -if [ -n "$COMMIT_TAG" ] && [ "$RELEASE" != "0" ] || [ "$BRANCH" = "master" ]; then +if [ "$BRANCH" == "master" ]; then exit 0 else exit 1 diff --git a/build-system/scripts/should_release b/build-system/scripts/should_release new file mode 100755 index 00000000000..5473fc6da76 --- /dev/null +++ b/build-system/scripts/should_release @@ -0,0 +1,9 @@ +#!/bin/bash +# Returns true if we are expected to proceed with a release job. +# Specifically if we have a commit tag, are master, or are being forced to release. +# This script should be used at the start of all release steps to early out PR runs. +# Later release steps may early out regardless, e.g. npm releases only run for tagged commits. +# Dockerhub publishing will publish for master, or PR's run with force-release, but only tag latest on tagged commits. +set -eu + +[[ -n "$COMMIT_TAG" || "$BRANCH" == "master" || "$COMMIT_MESSAGE" == *"[ci force-release]"* ]] \ No newline at end of file diff --git a/build_manifest.yml b/build_manifest.yml index 73c33d472de..588c3c70a93 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -1,10 +1,41 @@ noir: - # Add less aggressive rebuildPatterns. buildDir: noir + rebuildPatterns: + - ^noir/Dockerfile + - ^noir/acvm-repo + - ^noir/compiler + - ^noir/aztec_macros + - ^noir/noir_stdlib + - ^noir/tooling/backend_interface + - ^noir/tooling/bb_abstraction_leaks + - ^noir/tooling/debugger + - ^noir/tooling/lsp + - ^noir/tooling/nargo + - ^noir/tooling/nargo_cli + - ^noir/tooling/nargo_toml + - ^noir/tooling/nargo_fmt + - ^noir/tooling/noirc_abi + multiarch: host noir-packages: buildDir: noir dockerfile: Dockerfile.packages + rebuildPatterns: + - ^noir/Dockerfile.packages + - ^noir/.yarn + - ^noir/.yarnrc.yml + - ^noir/package.json + - ^noir/yarn.lock + - ^noir/acvm-repo + - ^noir/compiler + - ^noir/aztec_macros + - ^noir/noir_stdlib + - ^noir/tooling/noir_codegen + - ^noir/tooling/noir_js + - ^noir/tooling/noir_js_backend_barretenberg + - ^noir/tooling/noir_js_types + - ^noir/tooling/noirc_abi + - ^noir/tooling/noirc_abi_wasm noir-acir-tests: buildDir: noir/test_programs @@ -92,27 +123,38 @@ yarn-project: buildDir: yarn-project rebuildPatterns: - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml)$ - - ^yarn-project/Dockerfile + - ^yarn-project/Dockerfile$ dependencies: - yarn-project-base +yarn-project-prod: + buildDir: yarn-project + dockerfile: Dockerfile.prod + rebuildPatterns: + - ^yarn-project/Dockerfile.prod + dependencies: + - yarn-project + multiarch: buildx + aztec-sandbox: buildDir: yarn-project projectDir: yarn-project/aztec-sandbox dependencies: - - yarn-project + - yarn-project-prod + multiarch: buildx aztec-faucet: buildDir: yarn-project projectDir: yarn-project/aztec-faucet dependencies: - - yarn-project + - yarn-project-prod cli: buildDir: yarn-project projectDir: yarn-project/cli dependencies: - - yarn-project + - yarn-project-prod + multiarch: buildx boxes-blank-react: buildDir: yarn-project diff --git a/cspell.json b/cspell.json index 35d7779e39d..3d7775f13b2 100644 --- a/cspell.json +++ b/cspell.json @@ -21,6 +21,7 @@ "Bufferable", "bufs", "buildkit", + "buildx", "bytecodes", "calldatacopy", "callstack", @@ -93,6 +94,7 @@ "mplex", "msgpack", "muldiv", + "multiarch", "multivalue", "muxers", "Nargo", diff --git a/yarn-project/.dockerignore b/yarn-project/.dockerignore index 20f1f5070d4..c227696aa75 100644 --- a/yarn-project/.dockerignore +++ b/yarn-project/.dockerignore @@ -10,7 +10,7 @@ **/*.tsbuildinfo **/Dockerfile* **/node_modules -Dockerfile +Dockerfile* noir-contracts/src/types noir-contracts/src/artifacts diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod new file mode 100644 index 00000000000..82c21c1755c --- /dev/null +++ b/yarn-project/Dockerfile.prod @@ -0,0 +1,22 @@ +# This productionifies the workspace, removing all developer dependencies and producing a final slim image from which +# we then generate downstream multiarch containers to execute the specific projects. +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project + +# Need new arch specific image. +FROM node:18-alpine AS builder +RUN apk add bash jq --no-cache +COPY --from=yarn-project /usr/src /usr/src +WORKDIR /usr/src/yarn-project +ARG COMMIT_TAG="" +RUN ./scripts/version_packages.sh +# Productionify. See comment in yarn-project-base/Dockerfile. +RUN yarn workspaces focus @aztec/cli @aztec/aztec-sandbox @aztec/aztec-faucet --production && \ + yarn cache clean && \ + rm -rf ./**/src + +# We no longer need nargo. +RUN rm -rf /usr/src/noir/target + +# Create fresh minimal size image. +FROM node:18-alpine +COPY --from=builder /usr/src /usr/src \ No newline at end of file diff --git a/yarn-project/aztec-faucet/Dockerfile b/yarn-project/aztec-faucet/Dockerfile index d524e04c8ef..be0711dbb16 100644 --- a/yarn-project/aztec-faucet/Dockerfile +++ b/yarn-project/aztec-faucet/Dockerfile @@ -1,14 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder - +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod WORKDIR /usr/src/yarn-project/aztec-faucet - -# Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn cache clean && yarn workspaces focus --production - -# Create final, minimal size image. -FROM node:18-alpine -COPY --from=builder /usr/src/ /usr/src/ -WORKDIR /usr/src/yarn-project/aztec-faucet -ENTRYPOINT ["yarn"] -CMD [ "start" ] +ENTRYPOINT ["yarn", "start"] EXPOSE 8080 diff --git a/yarn-project/aztec-sandbox/Dockerfile b/yarn-project/aztec-sandbox/Dockerfile index 4dfc92a9596..40731ebdcab 100644 --- a/yarn-project/aztec-sandbox/Dockerfile +++ b/yarn-project/aztec-sandbox/Dockerfile @@ -1,34 +1,5 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod -# Need new arch specific image. -FROM node:18-alpine as builder -COPY --from=yarn-project /usr/src /usr/src -ARG COMMIT_TAG="" - -# Update pxe version if COMMIT_TAG has been used. -WORKDIR /usr/src/yarn-project/pxe -RUN if [[ -n "${COMMIT_TAG}" ]]; then \ - jq --arg v ${COMMIT_TAG} '.version = $v' package.json > _temp && mv _temp package.json; \ - fi - -# Update sandbox version if COMMIT_TAG has been used. -WORKDIR /usr/src/yarn-project/aztec-sandbox -RUN if [[ -n "${COMMIT_TAG}" ]]; then \ - jq --arg v ${COMMIT_TAG} '.version = $v' package.json > _temp && mv _temp package.json; \ - fi - -# Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn workspaces focus --production && yarn cache clean && rm -rf ../**/src - -# Create final, arch specific, minimal size image. -FROM node:18-alpine -COPY --from=builder /usr/src/yarn-project /usr/src/yarn-project -COPY --from=builder /usr/src/barretenberg/ts/package /usr/src/barretenberg/ts/package -COPY --from=builder /usr/src/noir/packages /usr/src/noir/packages -# Just until weird source-resolver bug fixed. -COPY --from=yarn-project /usr/src/noir/compiler /usr/src/noir/compiler WORKDIR /usr/src/yarn-project/aztec-sandbox -ENV NODE_OPTIONS=--preserve-symlinks -ENTRYPOINT ["yarn"] -CMD [ "start" ] +ENTRYPOINT ["yarn", "start"] EXPOSE 8079 8080 diff --git a/yarn-project/cli/Dockerfile b/yarn-project/cli/Dockerfile index 2d8c842c90f..0b8570793f7 100644 --- a/yarn-project/cli/Dockerfile +++ b/yarn-project/cli/Dockerfile @@ -1,33 +1,17 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod -# Need new arch specific image. -FROM node:18-alpine as builder -COPY --from=yarn-project /usr/src /usr/src -ARG COMMIT_TAG="" - -WORKDIR /usr/src/yarn-project/cli -RUN if [[ -n "${COMMIT_TAG}" ]]; then \ - jq --arg v ${COMMIT_TAG} '.version = $v' package.json > _temp && mv _temp package.json; \ - fi - -# Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn workspaces focus --production && yarn cache clean && rm -rf ../**/src - -# Create final, arch specific, minimal size image. -FROM node:18-alpine -COPY --from=builder /usr/src/yarn-project /usr/src/yarn-project -COPY --from=builder /usr/src/barretenberg/ts/package /usr/src/barretenberg/ts/package -COPY --from=builder /usr/src/noir/packages /usr/src/noir/packages +# Set Tini as the default entrypoint, to handle ctrl-c etc. +# Why not just yarn start? About 1 second difference in startup time. +# At time of writing it still takes 1.7s to just get the help to print. Needs investigating. +RUN apk add --no-cache tini +ENTRYPOINT ["/sbin/tini", "--", "node", "--no-warnings", "/usr/src/yarn-project/cli/dest/bin/index.js"] +# Setup cache volume. ENV XDG_CACHE_HOME /cache RUN mkdir /cache && chmod 777 /cache VOLUME [ "/cache" ] +# Run as non-root user. RUN corepack enable - -# run as non-root user RUN addgroup -S aztec && adduser -S aztec -G aztec USER aztec - -ENV NODE_OPTIONS="--no-warnings --preserve-symlinks" -ENTRYPOINT ["node", "/usr/src/yarn-project/cli/dest/bin/index.js"] \ No newline at end of file diff --git a/yarn-project/cli/src/utils.ts b/yarn-project/cli/src/utils.ts index 9973d9fa2fc..dc4f8732617 100644 --- a/yarn-project/cli/src/utils.ts +++ b/yarn-project/cli/src/utils.ts @@ -17,7 +17,7 @@ import { import { LogId } from '@aztec/types'; import { CommanderError, InvalidArgumentError } from 'commander'; -import fs from 'fs'; +import { readFile, rename, writeFile } from 'fs/promises'; import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; import { encodeArgs } from './encoding.js'; @@ -107,7 +107,7 @@ export async function getContractArtifact(fileDir: string, log: LogFn) { } try { - contents = fs.readFileSync(fileDir, 'utf8'); + contents = await readFile(fileDir, 'utf8'); } catch { throw Error(`Contract ${fileDir} not found`); } @@ -412,11 +412,11 @@ export function parseFields(fields: string[]): Fr[] { export async function atomicUpdateFile(filePath: string, contents: string) { const tmpFilepath = filePath + '.tmp'; try { - await fs.promises.writeFile(tmpFilepath, contents, { + await writeFile(tmpFilepath, contents, { // let's crash if the tmp file already exists flag: 'wx', }); - await fs.promises.rename(tmpFilepath, filePath); + await rename(tmpFilepath, filePath); } catch (e) { if (e instanceof Error && 'code' in e && e.code === 'EEXIST') { const commanderError = new CommanderError( diff --git a/yarn-project/deploy_npm.sh b/yarn-project/deploy_npm.sh index 2bd497870f4..029eb3a135b 100755 --- a/yarn-project/deploy_npm.sh +++ b/yarn-project/deploy_npm.sh @@ -2,9 +2,8 @@ [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu -# Check we're on a release flow. -if [ -z "$COMMIT_TAG" ] && [ ! "$DRY_DEPLOY" -eq 1 ]; then - echo "Not on a release flow, skipping deploy." +if [ -z "$COMMIT_TAG" ]; then + echo "No commit tag, not deploying to npm." exit 0 fi diff --git a/yarn-project/end-to-end/Dockerfile b/yarn-project/end-to-end/Dockerfile index 915eb134e5e..e3e8c489526 100644 --- a/yarn-project/end-to-end/Dockerfile +++ b/yarn-project/end-to-end/Dockerfile @@ -6,7 +6,7 @@ RUN yarn build:web WORKDIR /usr/src/yarn-project/end-to-end # Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn cache clean && yarn workspaces focus --production +RUN yarn workspaces focus --production && yarn cache clean # Create final, minimal size image. # TODO: Not very minimal as chromium adds about 500MB of bloat :/ Separate or install at test runtime? @@ -30,5 +30,4 @@ COPY --from=builder /usr/src/yarn-project/aztec.js/dest/main.js /usr/src/yarn-pr WORKDIR /usr/src/yarn-project/end-to-end -ENV NODE_OPTIONS=--preserve-symlinks ENTRYPOINT ["yarn", "test"] diff --git a/yarn-project/noir-contracts/scripts/types.sh b/yarn-project/noir-contracts/scripts/types.sh index c25603dda38..ebbc24bdc5f 100755 --- a/yarn-project/noir-contracts/scripts/types.sh +++ b/yarn-project/noir-contracts/scripts/types.sh @@ -41,10 +41,10 @@ process() { CONTRACT=$1 cd $ROOT - NODE_OPTIONS="--no-warnings --loader ts-node/esm" yarn ts-node --esm src/scripts/copy_source.ts $CONTRACT_NAME + node --no-warnings --loader ts-node/esm src/scripts/copy_source.ts $CONTRACT_NAME echo "Creating types for $CONTRACT" - NODE_OPTIONS="--no-warnings --loader ts-node/esm" yarn ts-node --esm src/scripts/copy_output.ts $CONTRACT_NAME + node --no-warnings --loader ts-node/esm src/scripts/copy_output.ts $CONTRACT_NAME } format() { diff --git a/yarn-project/scripts/version_packages.sh b/yarn-project/scripts/version_packages.sh new file mode 100755 index 00000000000..a708cb7a4ed --- /dev/null +++ b/yarn-project/scripts/version_packages.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -eu + +if [ -n "$COMMIT_TAG" ]; then + for workspace in $(yarn workspaces list --json | jq -r '.location'); do + (cd $workspace && jq --arg v $COMMIT_TAG '.version = $v' package.json > _temp && mv _temp package.json) + done +fi \ No newline at end of file diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index a4ee8c6b91e..a6ed174d487 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -66,7 +66,9 @@ RUN mkdir /usr/src/noir/compiler && mv /usr/src/noir/packages/source-resolver /u # We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they # walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution # correctly for portalled packages, is to use --preserve-symlinks when running node. -# This does work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic +# by initialising the module more than once. So at present I don't see a viable alternative. RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules WORKDIR /usr/src/yarn-project