From e2c543c7ec0483ede5e8c199776ad18c99e1a2e9 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 17:55:59 -0400 Subject: [PATCH] feat(ci): use content hash in build system, restrict docs build to *.ts or *.cpp (#1953) Benefits: - merges and squashes in your branch don't cause rebuilds - O(1) tag lookup, no need for tag renaming Should at least reduce the number of builds by half (stuff that passed will revalidate quickly on master etc) --- .circleci/config.yml | 5 +-- bootstrap_docker.sh | 4 +- build-system/remote_build/remote_build | 6 +-- build-system/scripts/build | 39 ++++++++----------- build-system/scripts/build_local | 2 +- build-system/scripts/calculate_content_hash | 17 ++++++++ build-system/scripts/changed | 11 ------ build-system/scripts/check_npm_version | 4 +- build-system/scripts/check_rebuild | 32 ++++----------- build-system/scripts/cond_spot_run_build | 23 ++++------- build-system/scripts/cond_spot_run_script | 18 ++++----- .../scripts/cond_spot_run_test_script | 4 +- build-system/scripts/cond_spot_run_tests | 2 +- build-system/scripts/deploy | 10 ++--- build-system/scripts/deploy_dockerhub | 4 +- build-system/scripts/deploy_ecr | 6 +-- build-system/scripts/deploy_global | 10 ++--- build-system/scripts/deploy_npm | 2 +- build-system/scripts/deploy_s3 | 4 +- build-system/scripts/deploy_service | 4 +- build-system/scripts/deploy_terraform | 4 +- build-system/scripts/ensure_apt_package | 4 +- build-system/scripts/ensure_repo | 4 +- build-system/scripts/ensure_terraform | 4 +- build-system/scripts/erase_image_tags | 7 ++-- build-system/scripts/extract_repo | 4 +- build-system/scripts/last_successful_commit | 18 --------- build-system/scripts/list_file_diff | 30 -------------- build-system/scripts/query_manifest | 3 +- build-system/scripts/remote_run_script | 12 +++--- build-system/scripts/request_spot | 6 +-- build-system/scripts/setup_env | 6 +-- build-system/scripts/spot_run_script | 21 +++++----- build-system/scripts/spot_run_test_script | 9 +++-- build-system/scripts/spot_run_tests | 2 +- .../scripts/store_test_benchmark_logs | 2 +- build-system/scripts/tag_remote_image | 4 +- build_manifest.json | 8 ++-- .../barretenberg/acir_tests/run_acir_tests.sh | 10 ++--- circuits/cpp/barretenberg/bootstrap_docker.sh | 4 +- .../cpp/barretenberg/cpp/bin-test/bin-test.sh | 2 +- circuits/cpp/barretenberg/cpp/format.sh | 2 +- .../cpp/barretenberg/cpp/scripts/bb-tests.sh | 10 ++--- .../cpp/barretenberg/cpp/scripts/bin-test.sh | 7 ++-- .../cpp/scripts/run_aztec_circuits_tests | 7 ++-- .../cpp/barretenberg/cpp/scripts/run_tests | 9 +++-- .../cpp/srs_db/download_ignition.sh | 4 +- .../sol/scripts/install_foundry.sh | 2 +- .../cpp/barretenberg/ts/scripts/run_tests | 4 +- circuits/cpp/format.sh | 2 +- .../cpp/scripts/build_run_tests_docker_local | 4 +- circuits/cpp/scripts/run_coverage | 2 +- circuits/cpp/scripts/run_tests | 7 ++-- circuits/cpp/scripts/run_tests_local | 2 +- circuits/cpp/scripts/tidy.sh | 2 +- l1-contracts/bootstrap.sh | 2 +- l1-contracts/scripts/install_foundry.sh | 2 +- scripts/ci/store_test_benchmark_logs | 8 ++-- scripts/git-subrepo/ext/bashplus/bin/bash+ | 2 +- .../git-subrepo/ext/bashplus/lib/bash+.bash | 2 +- .../ext/test-more-bash/ext/bashplus/bin/bash+ | 2 +- .../ext/bashplus/lib/bash+.bash | 2 +- scripts/git-subrepo/lib/git-subrepo | 4 +- .../lib/git-subrepo.d/help-functions.bash | 2 +- .../note/recreate-rebase-conflict.sh | 2 +- .../pkg/bin/generate-help-functions.pl | 2 +- scripts/git_subrepo.sh | 2 +- scripts/update.sh | 2 +- .../scripts/docker-compose-e2e-sandbox.yml | 2 +- yarn-project/canary/scripts/run_tests | 13 +++---- .../canary/scripts/update_packages.sh | 4 +- .../end-to-end/scripts/cond_run_script | 14 +++---- yarn-project/end-to-end/scripts/run_tests | 16 ++++---- .../end-to-end/scripts/run_tests_local | 16 ++++---- yarn-project/end-to-end/scripts/start_e2e.sh | 2 +- .../end-to-end/scripts/start_p2p_e2e.sh | 2 +- .../noir-contracts/scripts/install_noir.sh | 2 +- .../noir-contracts/scripts/install_noirup.sh | 2 +- .../noir-contracts/src/scripts/compile.sh | 2 +- .../p2p-bootstrap/scripts/start_bootnode.sh | 2 +- yarn-project/types/Dockerfile | 2 +- .../scripts/update_build_manifest.mjs | 5 ++- 82 files changed, 240 insertions(+), 313 deletions(-) create mode 100755 build-system/scripts/calculate_content_hash delete mode 100755 build-system/scripts/changed delete mode 100755 build-system/scripts/last_successful_commit delete mode 100755 build-system/scripts/list_file_diff diff --git a/.circleci/config.yml b/.circleci/config.yml index 16cea2556ab1..700f39f1f37b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,10 +57,8 @@ checkout: &checkout git remote add origin $CIRCLE_REPOSITORY_URL # Only download metadata when fetching. - retry_10 git fetch --depth 50 --filter=blob:none origin $CIRCLE_SHA1 + retry_10 git fetch --depth 1 --filter=blob:none origin $CIRCLE_SHA1 git checkout FETCH_HEAD - # Initialize submodules recursively (retry 10 times on failure) - retry_10 git submodule update --init --recursive # Called setup_env to setup a bunch of global variables used throughout the rest of the build process. # It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of @@ -1460,4 +1458,3 @@ workflows: requires: - build-deployment-canary <<: *deploy_defaults - diff --git a/bootstrap_docker.sh b/bootstrap_docker.sh index d4df940ff477..a8025f08aa12 100755 --- a/bootstrap_docker.sh +++ b/bootstrap_docker.sh @@ -32,11 +32,11 @@ if [ -z "$TARGET_PROJECT" ]; then fi fi -source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER $(git rev-parse --show-toplevel) +source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER build_local $TARGET_PROJECT $ONLY_TARGET if [ -z "$TARGET_PROJECT" ]; then echo echo "Success! You could now run e.g.:" echo " docker run -ti --rm aztecprotocol/end-to-end:latest e2e_private_token_contract.test" -fi \ No newline at end of file +fi diff --git a/build-system/remote_build/remote_build b/build-system/remote_build/remote_build index 60f8a2068478..8eab556ee578 100755 --- a/build-system/remote_build/remote_build +++ b/build-system/remote_build/remote_build @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts @@ -14,7 +14,7 @@ git remote add origin $GIT_REPOSITORY_URL # Only download metadata when fetching. git config remote.origin.promisor true git config remote.origin.partialclonefilter blob:none -git fetch --depth 50 origin $COMMIT_HASH +git fetch --depth 1 origin $COMMIT_HASH git checkout FETCH_HEAD # Checkout barretenberg submodule only. git submodule update --init build-system @@ -25,4 +25,4 @@ BASH_ENV=/tmp/bash_env echo "Calling setup env..." source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" echo "Calling build..." -build $@ \ No newline at end of file +build $@ diff --git a/build-system/scripts/build b/build-system/scripts/build index d830434eb30c..4add19d1f1a5 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -35,8 +35,6 @@ echo "Working directory: $PWD" echo "Dockerfile: $DOCKERFILE" echo "Build directory: $BUILD_DIR" -init_submodules $REPOSITORY - function fetch_image() { echo "Pulling: $1" if ! docker pull $1 > /dev/null 2>&1; then @@ -49,23 +47,20 @@ function fetch_image() { # Ensure ECR repository exists. ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY) -echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT" +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Content hash: $CONTENT_HASH" cd $BUILD_DIR # If we have previously successful commit, we can early out if nothing relevant has changed since. -if [[ $FORCE_BUILD == 'false' ]] && check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then - echo "No rebuild necessary. Retagging..." - STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') - for STAGE in $STAGES; do - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true - done - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH +if [[ $FORCE_BUILD == 'false' ]] && check_rebuild cache-"$CONTENT_HASH" $REPOSITORY; then + echo "No rebuild necessary." untag_remote_image $REPOSITORY tainted exit 0 fi +init_submodules $REPOSITORY + # Validate any terraform if it exists. if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then ensure_terraform @@ -94,13 +89,13 @@ PARENTS=$(cat $DOCKERFILE | sed -n -e "s/^FROM $ECR_DEPLOY_URL\/\([^[:space:]]\+ for PARENT in $PARENTS; do # Extract repository name (i.e. discard tag). PARENT_REPO=${PARENT%:*} - PARENT_COMMIT_HASH=$(last_successful_commit $PARENT_REPO) + PARENT_CONTENT_HASH=$(calculate_content_hash $PARENT_REPO) # There must be a parent image to continue. - if [ -z "$PARENT_COMMIT_HASH" ]; then + if [ -z "$PARENT_CONTENT_HASH" ]; then echo "No parent image found for $PARENT_REPO" exit 1 fi - PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:cache-$PARENT_COMMIT_HASH + PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:cache-$PARENT_CONTENT_HASH echo "Pulling dependency $PARENT_REPO..." fetch_image $PARENT_IMAGE_URI # Tag it to look like an official release as that's what we use in Dockerfiles. @@ -113,18 +108,18 @@ CACHE_FROM="" STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') for STAGE in $STAGES; do # Get the last build of this stage to leverage layer caching. - if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then + if [ -n "$CONTENT_HASH" ]; then echo "Pulling stage: $STAGE" - STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT-$STAGE + STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$STAGE if fetch_image $STAGE_IMAGE_LAST_URI; then STAGE_CACHE_FROM="--cache-from $STAGE_IMAGE_LAST_URI" fi fi echo "Building stage: $STAGE" - STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH-$STAGE + STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$STAGE # Build our dockerfile, add timing information - docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg ARG_COMMIT_HASH=$COMMIT_HASH . \ + docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . \ | while read line ; do echo "$(date "+%H:%M:%S")| $line"; done # We don't want to have redo this stages work when building the final image. Use it as a layer cache. @@ -136,8 +131,8 @@ for STAGE in $STAGES; do done # Pull previous image to use it as a layer cache if it exists. -if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then - LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT +if [ -n "$CONTENT_HASH" ]; then + LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH echo "Pulling previous build of $REPOSITORY..." fetch_image $LAST_SUCCESSFUL_URI || true CACHE_FROM="--cache-from $LAST_SUCCESSFUL_URI $CACHE_FROM" @@ -145,10 +140,10 @@ if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then fi # Build the actual image and give it a commit tag. -IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH +IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH echo "Building image: $IMAGE_COMMIT_URI" # Build our dockerfile, add timing information -docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG --build-arg ARG_COMMIT_HASH=$COMMIT_HASH . \ +docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . \ | while read line ; do echo "$(date "+%H:%M:%S")| $line"; done echo "Pushing image: $IMAGE_COMMIT_URI" docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index 9df4d1d0e0af..bb1d63408448 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -6,7 +6,7 @@ # If DOCKERFILE is excluded it tries to default to ./Dockerfile then .//Dockerfile # If REPO is excluded it defaults to PROJECT_DIR_NAME. -set -e +set -eu TARGET_PROJECT=$1 ONLY_TARGET=$2 diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash new file mode 100755 index 000000000000..0ecc960521e6 --- /dev/null +++ b/build-system/scripts/calculate_content_hash @@ -0,0 +1,17 @@ +#!/bin/bash + +set -eu + +REPOSITORY=$1 +COMMIT_HASH=${2:-$COMMIT_HASH} + +# Compute REBUILD_PATTERNS from the build manifest +REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) + +AWK_PATTERN=$(echo $REBUILD_PATTERNS | sed 's/ /|/g') +# an example line is +# 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json +# this format is beneficial as it grabs the hashes from git efficiently +# we will next filter by our rebuild patterns +# then we pipe the hash portion of each file to git hash-object to produce our content hash +git ls-tree -r HEAD | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin diff --git a/build-system/scripts/changed b/build-system/scripts/changed deleted file mode 100755 index fb2358ff1196..000000000000 --- a/build-system/scripts/changed +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# Returns true if the file at path has changed since the LAST_SUCCESSFUL_COMMIT. -LAST_SUCCESSFUL_COMMIT=$1 - -[ -n "$COMMIT_HASH" ] || exit 1 - -if git diff --name-only $LAST_SUCCESSFUL_COMMIT $COMMIT_HASH | grep -q $2; then - exit 0 -else - exit 1 -fi \ No newline at end of file diff --git a/build-system/scripts/check_npm_version b/build-system/scripts/check_npm_version index ee61fd5bccf0..0d7de47828ec 100755 --- a/build-system/scripts/check_npm_version +++ b/build-system/scripts/check_npm_version @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu readonly LOCAL_VERSION=$(node -pe "require('./package.json').version") readonly PACKAGE_NAME=${1:-./} @@ -14,4 +14,4 @@ elif [ "$LOCAL_VERSION" != "$PUBLISHED_VERSION" ] && [ "$LOCAL_VERSION" == "$HIG else echo "Expect npm version number to be higher than '$PUBLISHED_VERSION'. Current local version is '$LOCAL_VERSION'." exit 1 -fi \ No newline at end of file +fi diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index 5093e796b0bd..d80e94936846 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -4,36 +4,20 @@ # The rebuild patterns are taken from the build manifest (computed from set of dependencies). set -euo pipefail -BASE_COMMIT=$1 +TAG=$1 REPOSITORY=$2 # If given nothing, then exit with failure to rebuild -[ -n "$BASE_COMMIT" ] || exit 1 +[ -n "$TAG" ] || exit 1 -# If a tainted tag exists, remove it exit with failure to rebuild. -if image_exists $REPOSITORY tainted; then - echo "$REPOSITORY has been tainted. Will rebuild." - exit 1 -fi - -# Compute .rebuild_patterns from the build manifest. -query_manifest rebuildPatterns $REPOSITORY > .rebuild_patterns - -echo "Rebuild patterns:" -cat .rebuild_patterns - -git config diff.renameLimit 999999 - -# Get list of files that differ -different_files=$(list_file_diff ${BASE_COMMIT} ${COMMIT_HASH}) || { - echo "list_file_diff failed. Rebuild required."; - exit 1; -} - -if grep -f .rebuild_patterns <<< "$different_files" &> /dev/null; then +if ! image_exists $REPOSITORY $TAG; then echo "Rebuild required." exit 1 +elif image_exists $REPOSITORY tainted; then + # If a tainted tag exists, remove it exit with failure to rebuild. + echo "$REPOSITORY has been tainted. Will rebuild." + exit 1 else echo "No rebuild required." exit 0 -fi \ No newline at end of file +fi diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 4e9b38ff7516..61f262307e5e 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu set -o pipefail REPOSITORY=$1 @@ -8,20 +8,13 @@ SPEC=$1 shift DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) -init_submodules $REPOSITORY - -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY) -echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT" +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Content hash: $CONTENT_HASH" cd $(query_manifest buildDir $REPOSITORY) -if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then - spot_run_script $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@ -else - echo "No rebuild necessary. Retagging..." - STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') - for STAGE in $STAGES; do - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true - done - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH -fi \ No newline at end of file +if ! check_rebuild cache-$CONTENT_HASH $REPOSITORY; then + init_submodules $REPOSITORY + spot_run_script $CONTENT_HASH $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@ + tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH +fi diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index fa12e39493e4..154b70c8b684 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -11,21 +11,21 @@ # # Arguments are: # 1. REPOSITORY: The project repository name in ECR. Used to determine if there are changes since last success. -# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a +# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a prefix after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Arguments to pass to spot_run_script. -set -e +set -eu REPOSITORY=$1 shift SUCCESS_TAG=$1 shift -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $SUCCESS_TAG) +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Content hash: $CONTENT_HASH" -echo "Last successful commit for $SUCCESS_TAG: $LAST_SUCCESSFUL_COMMIT" - -if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then - spot_run_script $@ - tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$SUCCESS_TAG -fi \ No newline at end of file +if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then + init_submodules $REPOSITORY + spot_run_script $CONTENT_HASH $@ + tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG +fi diff --git a/build-system/scripts/cond_spot_run_test_script b/build-system/scripts/cond_spot_run_test_script index d4bccd922cd0..c760e69d00fc 100755 --- a/build-system/scripts/cond_spot_run_test_script +++ b/build-system/scripts/cond_spot_run_test_script @@ -1,12 +1,10 @@ #!/bin/bash -set -e +set -eu SCRIPT_PATH=$1 REPOSITORY=$2 shift shift -init_submodules $REPOSITORY - cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs diff --git a/build-system/scripts/cond_spot_run_tests b/build-system/scripts/cond_spot_run_tests index 07734f637a8c..78097379f1f9 100755 --- a/build-system/scripts/cond_spot_run_tests +++ b/build-system/scripts/cond_spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -e +set -eu cond_spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/deploy b/build-system/scripts/deploy index d8974f3a3e92..ae6d92a69ba8 100755 --- a/build-system/scripts/deploy +++ b/build-system/scripts/deploy @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu REPOSITORY=$1 SERVICES=${2:-$REPOSITORY} @@ -11,9 +11,9 @@ cd $(query_manifest projectDir $REPOSITORY) deploy_ecr $REPOSITORY # Bail out if nothing changed. -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $DEPLOY_TAG-deployed) -echo "Last successfully deployed commit: $LAST_SUCCESSFUL_COMMIT" -if check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Last successfully deployed commit: $CONTENT_HASH" +if check_rebuild cache-$CONTENT_HASH-$DEPLOY_TAG-deployed $REPOSITORY; then echo "No changes detected, skipping deployment." exit 0 fi @@ -26,4 +26,4 @@ for SERVICE in $SERVICES; do done # Tag the image as deployed. -tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$DEPLOY_TAG-deployed \ No newline at end of file +tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 1f1db34287f3..d7e0442d30b7 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu if [ -z "$COMMIT_TAG" ]; then echo "Will only push tagged builds to dockerhub. Skipping." @@ -56,4 +56,4 @@ docker tag $IMAGE_COMMIT_URI $IMAGE_LATEST_URI # Push tagged image to dockerhub. docker push $IMAGE_DEPLOY_URI # Push :latest image to dockerhub -docker push $IMAGE_LATEST_URI \ No newline at end of file +docker push $IMAGE_LATEST_URI diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr index e149ec09ae7b..aa512cc1b4bf 100755 --- a/build-system/scripts/deploy_ecr +++ b/build-system/scripts/deploy_ecr @@ -1,8 +1,8 @@ #!/bin/bash -set -e +set -eu REPOSITORY=$1 -IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH +IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH # Login to build region and pull the build. ensure_repo $REPOSITORY $ECR_REGION @@ -22,4 +22,4 @@ if [ -n "$COMMIT_TAG" ]; then tag_remote_image $REPOSITORY $COMMIT_HASH $PROJECT-$COMMIT_TAG $ECR_DEPLOY_REGION fi -tag_remote_image $REPOSITORY $COMMIT_HASH $DEPLOY_TAG $ECR_DEPLOY_REGION \ No newline at end of file +tag_remote_image $REPOSITORY $COMMIT_HASH $DEPLOY_TAG $ECR_DEPLOY_REGION diff --git a/build-system/scripts/deploy_global b/build-system/scripts/deploy_global index 31cdef016131..5114a067a74c 100755 --- a/build-system/scripts/deploy_global +++ b/build-system/scripts/deploy_global @@ -1,6 +1,6 @@ #!/bin/bash # Deployment script for global service (e.g. company website and metrics). -set -e +set -eu REPOSITORY=$1 @@ -10,9 +10,9 @@ cd $(query_manifest projectDir $REPOSITORY) deploy_ecr $REPOSITORY # Bail out if nothing changed. -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $DEPLOY_TAG-deployed) -echo "Last successfully deployed commit: $LAST_SUCCESSFUL_COMMIT" -if check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Last successfully deployed commit: $CONTENT_HASH" +if check_rebuild cache-$CONTENT_HASH-$DEPLOY_TAG-deployed $REPOSITORY; then echo "No changes detected, skipping deployment." exit 0 fi @@ -22,4 +22,4 @@ deploy_terraform "" ./terraform deploy_service $REPOSITORY # Tag the image as deployed. -tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$DEPLOY_TAG-deployed \ No newline at end of file +tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed diff --git a/build-system/scripts/deploy_npm b/build-system/scripts/deploy_npm index 711856333700..1847e2463d10 100755 --- a/build-system/scripts/deploy_npm +++ b/build-system/scripts/deploy_npm @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu readonly REPOSITORY=$1 readonly STANDALONE=$2 diff --git a/build-system/scripts/deploy_s3 b/build-system/scripts/deploy_s3 index 9b000dfa58e9..947062da2068 100755 --- a/build-system/scripts/deploy_s3 +++ b/build-system/scripts/deploy_s3 @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu REPOSITORY=$1 # A global deploy, doesn't namespace under a deploy tag (e.g. company website, metrics) @@ -46,4 +46,4 @@ aws s3 cp \ if [ -n "$AWS_CLOUDFRONT_DISTRIBUTION" ]; then aws cloudfront create-invalidation --distribution-id $AWS_CLOUDFRONT_DISTRIBUTION --paths "/*" -fi \ No newline at end of file +fi diff --git a/build-system/scripts/deploy_service b/build-system/scripts/deploy_service index 37d9e728884a..c126becdf562 100755 --- a/build-system/scripts/deploy_service +++ b/build-system/scripts/deploy_service @@ -1,8 +1,8 @@ #!/bin/bash -set -e +set -eu # Redeploy service with latest image. SERVICE_NAME=$1 if aws ecs list-services --region $ECR_DEPLOY_REGION --cluster setup | grep "/$SERVICE_NAME\"" > /dev/null; then aws ecs update-service --region $ECR_DEPLOY_REGION --cluster setup --service $SERVICE_NAME --force-new-deployment -fi \ No newline at end of file +fi diff --git a/build-system/scripts/deploy_terraform b/build-system/scripts/deploy_terraform index 00e0ad22aebb..637ef5c8be17 100755 --- a/build-system/scripts/deploy_terraform +++ b/build-system/scripts/deploy_terraform @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu REPOSITORY=$1 TF_DIR=$2 @@ -38,4 +38,4 @@ for RESOURCE in $TO_TAINT; do terraform taint $RESOURCE || true done -terraform apply -input=false -auto-approve \ No newline at end of file +terraform apply -input=false -auto-approve diff --git a/build-system/scripts/ensure_apt_package b/build-system/scripts/ensure_apt_package index e546752c23a7..33326c7f1fd7 100755 --- a/build-system/scripts/ensure_apt_package +++ b/build-system/scripts/ensure_apt_package @@ -1,9 +1,9 @@ #!/bin/bash -set -e +set -eu if dpkg -s $1 &> /dev/null; then exit 0 fi >&2 echo "Installing $1..." -sudo apt update &> /dev/null && sudo apt install --force-yes $1 &> /dev/null \ No newline at end of file +sudo apt update &> /dev/null && sudo apt install --force-yes $1 &> /dev/null diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index 31089f8f64c0..186b204cd4e9 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -1,7 +1,7 @@ #!/bin/bash # Logs the shell into the ECR instance at the given region, establishes if the given repository exists, creates it if it # doesn't, and re-applies thie lifecycle policy (determines when images should be automatically deleted) if it does. -set -e +set -eu LIFECYCLE_POLICY='{ "rules": [ @@ -37,4 +37,4 @@ elif [ -n "$REFRESH_LIFECYCLE" ]; then echo "Refreshing lifecycle rules for repo: $REPOSITORY" aws ecr put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 fi -echo \ No newline at end of file +echo diff --git a/build-system/scripts/ensure_terraform b/build-system/scripts/ensure_terraform index c1686f38f647..e04231e79f13 100755 --- a/build-system/scripts/ensure_terraform +++ b/build-system/scripts/ensure_terraform @@ -1,6 +1,6 @@ #!/bin/bash # Downloads and installs `terraform` if it's not installed. -set -e +set -eu [ ! -f /usr/local/bin/terraform ] || exit 0 @@ -10,4 +10,4 @@ curl -sSL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terrafor sudo apt install -y unzip unzip terraform.zip sudo mv terraform /usr/local/bin/ -rm terraform.zip \ No newline at end of file +rm terraform.zip diff --git a/build-system/scripts/erase_image_tags b/build-system/scripts/erase_image_tags index 930f4d4d7894..4c61679f835c 100755 --- a/build-system/scripts/erase_image_tags +++ b/build-system/scripts/erase_image_tags @@ -1,7 +1,7 @@ #!/bin/bash # Erase the image tag associated with the last commit for the given repository. # If TILL_COMMIT_HASH is given, erase tags going back in time until we reach TILL_COMMIT_HASH. -set -e +set -eu REPOSITORY=$1 TILL_COMMIT_HASH=$2 @@ -11,7 +11,8 @@ if [ -n "$3" ]; then fi for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do - TAG=cache-${COMMIT_HASH}$TAG_POSTFIX + CONTENT_HASH=$(calculate_content_hash $REPOSITORY $COMMIT_HASH) + TAG=cache-${CONTENT_HASH}$TAG_POSTFIX if image_exists $REPOSITORY $TAG; then echo "Erasing image tag: $REPOSITORY:$TAG" aws ecr batch-delete-image --region=$ECR_REGION --repository-name $1 --image-ids imageTag=$TAG > /dev/null @@ -19,4 +20,4 @@ for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do break fi fi -done \ No newline at end of file +done diff --git a/build-system/scripts/extract_repo b/build-system/scripts/extract_repo index 64576acae4c5..8bd7ef2c6068 100755 --- a/build-system/scripts/extract_repo +++ b/build-system/scripts/extract_repo @@ -1,6 +1,6 @@ #!/bin/bash # Given a repository, extracts the builds entire /usr/src dir to the given path. -set -e +set -eu REPOSITORY=$1 EXTRACT_FROM=${2:-/usr/src} @@ -17,4 +17,4 @@ docker cp $TEMP_CONTAINER:$EXTRACT_FROM $EXTRACT_TO docker rm -v $TEMP_CONTAINER > /dev/null echo "Extracted contents:" -ls -al $EXTRACT_TO \ No newline at end of file +ls -al $EXTRACT_TO diff --git a/build-system/scripts/last_successful_commit b/build-system/scripts/last_successful_commit deleted file mode 100755 index 03b0fe625add..000000000000 --- a/build-system/scripts/last_successful_commit +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -REPOSITORY=$1 - -if [ -n "$2" ]; then - TAG_POSTFIX=-$2 -fi - -# We are assuming that there has been a successful build of the given repo -# within the last 50 commits. If not return nothing. -for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do - >&2 echo "Checking if image exists for commit $COMMIT_HASH..." - if image_exists $REPOSITORY cache-${COMMIT_HASH}$TAG_POSTFIX; then - >&2 echo "Found." - echo $COMMIT_HASH - exit 0 - fi -done \ No newline at end of file diff --git a/build-system/scripts/list_file_diff b/build-system/scripts/list_file_diff deleted file mode 100755 index d0721fd73513..000000000000 --- a/build-system/scripts/list_file_diff +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Lists files that have changed between two commits, including files in submodules. -# Needed because `git diff --submodule=diff --name-only` doesn't work as one would hope. -set -euo pipefail - -BASE_COMMIT=$1 -COMMIT_HASH=${2:-HEAD} - -if [ -z "$BASE_COMMIT" ]; then - echo "Usage $0 from_commit [to_commit]" - exit 1 -fi - -cd $ROOT_PATH - -git --no-pager diff --name-only ${BASE_COMMIT} ${COMMIT_HASH} - -for SUBMODULE_PATH in $(git config --file .gitmodules --get-regexp path | awk '{ print $2 }'); do - [ -f $SUBMODULE_PATH/.git ] || continue - if [ "$(git ls-tree $BASE_COMMIT $SUBMODULE_PATH | awk '{print $2}')" = "tree" ]; then - # In the base commit, we were not a submodule, and now we are. Output all files as having changed. - (cd $SUBMODULE_PATH && - git ls-files | awk "\$0=\"$SUBMODULE_PATH/\"\$0") - continue - fi - SM_PREV_HASH=$(git ls-tree $BASE_COMMIT $SUBMODULE_PATH | awk '{print $3}') - SM_CURR_HASH=$(git ls-tree $COMMIT_HASH $SUBMODULE_PATH | awk '{print $3}') - (cd $SUBMODULE_PATH && - git --no-pager diff --name-only ${SM_PREV_HASH} ${SM_CURR_HASH} | awk "\$0=\"$SUBMODULE_PATH/\"\$0") -done diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index f968f8fb57ca..ba7b8c1fe337 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -1,6 +1,5 @@ #!/bin/bash -set -e - +set -eu CMD=$1 REPO=$2 diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index f393e2649daf..c33d386f1f90 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -1,10 +1,10 @@ #!/bin/bash -set -e +set -eu IP=$1 -FULL_PATH=$2 -shift -shift +CONTENT_HASH=$2 +FULL_PATH=$3 +shift 3 SSH_CONFIG_PATH=${SSH_CONFIG_PATH:-$BUILD_SYSTEM_PATH/remote/ssh_config} DIR_NAME=$(dirname $FULL_PATH) @@ -13,5 +13,5 @@ SCRIPT_NAME=$(basename $FULL_PATH) # Copy all files in script directory to spot instance. scp -F $SSH_CONFIG_PATH $DIR_NAME/* $IP:. -# Run script on remote instance. -ssh -A -F $SSH_CONFIG_PATH $IP "COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" +# Run script on remote instance, passing environment variables. +ssh -A -F $SSH_CONFIG_PATH $IP "CONTENT_HASH=$CONTENT_HASH COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index 42f2faaf90d8..d87fd829eee6 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu NAME=$1 CPUS=${2:-32} @@ -51,7 +51,7 @@ fi aws ec2 create-tags --resources $IID --tags "Key=Name,Value=$NAME" aws ec2 create-tags --resources $IID --tags "Key=Group,Value=build-instance" -while [ -z "$IP" ]; do +while [ -z "${IP:-}" ]; do sleep 1 IP=$(aws ec2 describe-instances \ --filter "Name=instance-id,Values=$IID" \ @@ -63,4 +63,4 @@ done >&2 echo "Waiting for SSH at $IP..." while ! nc -z $IP 22; do sleep 1; done; -echo $IP \ No newline at end of file +echo $IP diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 0cb8e22f8d98..2df97fbfd5a2 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -6,13 +6,13 @@ # The script should be sourced from the root of the repository, e.g: # source ./build-system/scripts/setup_env # This ensures the resultant variables are set in the calling shell. -set -e +set -eu COMMIT_HASH=$1 COMMIT_TAG=$2 JOB_NAME=$3 -GIT_REPOSITORY_URL=$4 -BRANCH=$5 +GIT_REPOSITORY_URL=${4:-} +BRANCH=${5:-} BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) PROJECT=$(cat PROJECT) diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 1f3ee311fed1..63f16928c857 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -1,28 +1,29 @@ #!/bin/bash # Runs a test script on a remote spot instance. Arguments are: -# 1. SPEC: Instance specification filename. -# 2... ARGS: Arguments to pass to remote_run_script. -set -e +# 1. Content hash for our repository contents. Used for identify spot jobs and image tags. +# 2. SPEC: Instance specification filename. +# 3... ARGS: Arguments to pass to remote_run_script. +set -eu +CONTENT_HASH=$1 +SPEC=$2 +shift 2 # On any sort of exit (error or not), kill spot request so it doesn't count against quota. function on_exit { - if [ -f "sir-$COMMIT_HASH:$JOB_NAME.txt" ]; then - SIR=$(cat sir-$COMMIT_HASH:$JOB_NAME.txt) + if [ -f "sir-$CONTENT_HASH:$JOB_NAME.txt" ]; then + SIR=$(cat sir-$CONTENT_HASH:$JOB_NAME.txt) echo "Cancelling spot instance request $SIR (silently)" aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $SIR >/dev/null 2>&1 || true fi } trap on_exit EXIT -SPEC=$1 -shift - # Get spot instance. -IP=$(request_spot $COMMIT_HASH:$JOB_NAME $SPEC) +IP=$(request_spot $CONTENT_HASH:$JOB_NAME $SPEC) # Run script remotely on spot instance, capturing success or failure. set +e -remote_run_script $IP $@ +remote_run_script $IP $CONTENT_HASH $@ CODE=$? # Shutdown spot. diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script index 0fb7e5104346..06d4d4553805 100755 --- a/build-system/scripts/spot_run_test_script +++ b/build-system/scripts/spot_run_test_script @@ -1,15 +1,16 @@ #!/bin/bash -set -e +set -eu SCRIPT_PATH=$1 REPOSITORY=$2 shift shift -init_submodules $REPOSITORY - cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs set -o pipefail -spot_run_script 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" + +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Content hash: $CONTENT_HASH" +spot_run_script $CONTENT_HASH 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/build-system/scripts/spot_run_tests b/build-system/scripts/spot_run_tests index df2f738473bc..e7d8183c0480 100755 --- a/build-system/scripts/spot_run_tests +++ b/build-system/scripts/spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -e +set -eu spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/store_test_benchmark_logs b/build-system/scripts/store_test_benchmark_logs index 424a9ff31889..d44f3c2cd9fe 100755 --- a/build-system/scripts/store_test_benchmark_logs +++ b/build-system/scripts/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu REPOSITORY=$1 shift diff --git a/build-system/scripts/tag_remote_image b/build-system/scripts/tag_remote_image index ee42f32cb879..741009ff1251 100755 --- a/build-system/scripts/tag_remote_image +++ b/build-system/scripts/tag_remote_image @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu REPOSITORY=$1 EXISTING_TAG=$2 @@ -32,4 +32,4 @@ if [ "$EXISTING_TAG_MANIFEST" != "$NEW_TAG_MANIFEST" ]; then --repository-name $REPOSITORY \ --image-tag $NEW_TAG \ --image-manifest "$EXISTING_TAG_MANIFEST" > /dev/null 2>&1 -fi \ No newline at end of file +fi diff --git a/build_manifest.json b/build_manifest.json index 1f862827391b..c4b3b24cdacc 100644 --- a/build_manifest.json +++ b/build_manifest.json @@ -126,7 +126,9 @@ "buildDir": ".", "dockerfile": "docs/Dockerfile", "rebuildPatterns": [ - "^." + "^docs/", + "^.*.cpp$", + "^.*.ts$" ], "dependencies": [] }, @@ -282,7 +284,6 @@ ], "dependencies": [ "aztec.js", - "cli", "foundation", "l1-artifacts", "noir-contracts" @@ -314,7 +315,6 @@ "aztec-sandbox", "aztec.js", "circuits.js", - "cli", "ethereum", "foundation", "l1-artifacts", @@ -525,4 +525,4 @@ "types" ] } -} \ No newline at end of file +} diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index 1deabdaf3e91..673677562203 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -3,7 +3,7 @@ # BB: to specify a different binary to test with (e.g. bb.js or bb.js-dev). # VERBOSE: to enable logging for each test. -set -e +set -eu BB=$PWD/${BB:-../cpp/build/bin/bb} CRS_PATH=~/.bb-crs @@ -11,7 +11,7 @@ BRANCH=kw/acvm-0-24 # Pull down the test vectors from the noir repo, if we don't have the folder already. if [ ! -d acir_tests ]; then - if [ -n "$TEST_SRC" ]; then + if [ -n "${TEST_SRC:-}" ]; then cp -R $TEST_SRC acir_tests else rm -rf noir @@ -48,13 +48,13 @@ function test() { cd $1 set +e - if [ -n "$VERBOSE" ]; then + if [ -n "${VERBOSE:-}" ]; then $BB prove_and_verify -v -c $CRS_PATH -b ./target/$dir_name.bytecode else $BB prove_and_verify -c $CRS_PATH -b ./target/$dir_name.bytecode > /dev/null 2>&1 fi result=$? - set -e + set -xeu if [ $result -eq 0 ]; then echo -e "\033[32mPASSED\033[0m" @@ -68,7 +68,7 @@ function test() { cd .. } -if [ -n "$1" ]; then +if [ -n "${1:-}" ]; then test $1 else for DIR in $(find -maxdepth 1 -type d -not -path '.'); do diff --git a/circuits/cpp/barretenberg/bootstrap_docker.sh b/circuits/cpp/barretenberg/bootstrap_docker.sh index 737f9d075633..e2f88e225253 100755 --- a/circuits/cpp/barretenberg/bootstrap_docker.sh +++ b/circuits/cpp/barretenberg/bootstrap_docker.sh @@ -1,8 +1,8 @@ #!/bin/bash # This script builds the projects listed in build_mainifest.sh. -set -e +set -eu COMMIT_HASH=$(git rev-parse HEAD) source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER $(git rev-parse --show-toplevel) -build_local \ No newline at end of file +build_local diff --git a/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh b/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh index 756c130ee8cd..eb634c960652 100755 --- a/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh @@ -9,4 +9,4 @@ bb() { bb gates bb prove -o proof bb write_vk -o vk -bb verify -k vk -p proof \ No newline at end of file +bb verify -k vk -p proof diff --git a/circuits/cpp/barretenberg/cpp/format.sh b/circuits/cpp/barretenberg/cpp/format.sh index 20c99bf89938..48315e39621c 100755 --- a/circuits/cpp/barretenberg/cpp/format.sh +++ b/circuits/cpp/barretenberg/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index b7dd5c27b8b4..235aa111ff4c 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -1,12 +1,12 @@ #!/bin/bash # This script runs all test suites that have not been broken out into their own jobs for parallelisation. # Might be better to list exclusions here rather than inclusions as risky to maintain. -set -e +set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert:cache-$COMMIT_HASH - +REPOSITORY=barretenberg-x86_64-linux-clang-assert +# use the image rebuild patterns to compute a content hash, use this to get a URI +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI TESTS=( @@ -30,7 +30,7 @@ TESTS=( TESTS_STR="${TESTS[@]}" docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/barretenberg/cpp; \ (cd srs_db && ./download_ignition.sh 1); \ cd build; \ diff --git a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh index b344ea40025e..b06c26036bac 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh @@ -3,12 +3,11 @@ set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert:cache-$COMMIT_HASH - +REPOSITORY=barretenberg-x86_64-linux-clang-assert +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/barretenberg/cpp/bin-test; \ ./bin-test.sh" diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index 0c3358300cdb..d67176f098fa 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. @@ -19,7 +19,8 @@ shift # to aztec's circuits `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-circuits-${ARCH}-linux-clang-builder-runner:cache-$COMMIT_HASH +REPOSITORY="barretenberg-circuits-${ARCH}-linux-clang-builder-runner" +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI if [ "$ARCH" != "wasm" ]; then @@ -36,7 +37,7 @@ echo "*** Running Aztec circuits tests on commit: $AZTEC_COMMIT" # run tests in docker RUN_ARGS="$@" # helper var necessary for some reason to pass all args to docker run docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/; \ git clone https://github.com/AztecProtocol/aztec3-packages.git; \ cd /usr/src/aztec3-packages/circuits/cpp; \ diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 409379f4e015..965307ff5a59 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -3,7 +3,7 @@ # 1. The number of ignition transcripts to download. # 2. The set of gtest binary names to run. # 3-n. The arguments to pass to the gtest binaries. -set -e +set -eu NUM_TRANSCRIPTS=$1 TESTS=$2 @@ -12,8 +12,9 @@ shift $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert:cache-$COMMIT_HASH - +# use the image rebuild patterns to compute a content hash, use this to get a URI +REPOSITORY=barretenberg-x86_64-linux-clang-assert +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI if [ -f "$TESTS" ]; then @@ -21,7 +22,7 @@ if [ -f "$TESTS" ]; then fi docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/barretenberg/cpp/srs_db; \ ./download_ignition.sh $NUM_TRANSCRIPTS; \ cd /usr/src/barretenberg/cpp/build; \ diff --git a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh index 1632c0357811..478f442f9afd 100755 --- a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh +++ b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh @@ -13,7 +13,7 @@ # If a checksums file is available, it will be used to validate if a download is required # and also check the validity of the downloaded transcripts. If not the script downloads # whatever is requested but does not check the validity of the downloads. -set -e +set -eu mkdir -p ignition cd ignition @@ -45,4 +45,4 @@ for TRANSCRIPT in $(seq 0 $NUM); do else download $NUM fi -done \ No newline at end of file +done diff --git a/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh b/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh index 3842f6a4318b..e085beb76d59 100755 --- a/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh +++ b/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh @@ -18,4 +18,4 @@ chmod +x $BIN_PATH export PATH=$FOUNDRY_BIN_DIR:$PATH # Use version. -foundryup \ No newline at end of file +foundryup diff --git a/circuits/cpp/barretenberg/ts/scripts/run_tests b/circuits/cpp/barretenberg/ts/scripts/run_tests index aad403925c46..cd417d0724a7 100755 --- a/circuits/cpp/barretenberg/ts/scripts/run_tests +++ b/circuits/cpp/barretenberg/ts/scripts/run_tests @@ -2,5 +2,5 @@ set -e $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/bb.js:cache-$COMMIT_HASH -docker run --rm $IMAGE_URI \ No newline at end of file +IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/bb.js:cache-$CONTENT_HASH +docker run --rm $IMAGE_URI diff --git a/circuits/cpp/format.sh b/circuits/cpp/format.sh index 20c99bf89938..48315e39621c 100755 --- a/circuits/cpp/format.sh +++ b/circuits/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/scripts/build_run_tests_docker_local b/circuits/cpp/scripts/build_run_tests_docker_local index 3bf05cf8363c..a075cbba1727 100755 --- a/circuits/cpp/scripts/build_run_tests_docker_local +++ b/circuits/cpp/scripts/build_run_tests_docker_local @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu # To be called only LOCALLY for testing WITH docker. # Builds a docker image and runs tests in it. @@ -31,7 +31,7 @@ time docker build -f $DOCKERFILE -t $IMAGE_URI . # run tests in docker image RUN_ARGS="$@" # helper var necessary for some reason to pass all args to docker run time docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/circuits/cpp/barretenberg/cpp/srs_db; \ ln -sf /usr/src/circuits/cpp/barretenberg/cpp/srs_db /usr/src/circuits/cpp/srs_db; \ ./download_ignition.sh $NUM_TRANSCRIPTS; \ diff --git a/circuits/cpp/scripts/run_coverage b/circuits/cpp/scripts/run_coverage index 174d11ac6342..d0e4bb097f45 100755 --- a/circuits/cpp/scripts/run_coverage +++ b/circuits/cpp/scripts/run_coverage @@ -1,5 +1,5 @@ #!bin/bash -set -e +set -eu # To be called LOCALLY for testing WITHOUT docker. # diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index 13f679a3ab6d..b3fbc43f4592 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. @@ -25,13 +25,14 @@ shift # arg1 (num transcripts) is not forwarded to `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/circuits-$ARCH-linux-clang-assert:cache-$COMMIT_HASH +REPOSITORY="circuits-$ARCH-linux-clang-assert" +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI # run tests in docker image RUN_ARGS="$@" # helper var necessary for some reason to pass all args to docker run time docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/circuits/cpp/barretenberg/cpp/srs_db; \ ln -sf /usr/src/circuits/cpp/barretenberg/cpp/srs_db /usr/src/circuits/cpp/srs_db; \ ./download_ignition.sh $NUM_TRANSCRIPTS; \ diff --git a/circuits/cpp/scripts/run_tests_local b/circuits/cpp/scripts/run_tests_local index e209cda65cfe..4c942a0f13a4 100755 --- a/circuits/cpp/scripts/run_tests_local +++ b/circuits/cpp/scripts/run_tests_local @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu DIR="$(dirname "$0")" diff --git a/circuits/cpp/scripts/tidy.sh b/circuits/cpp/scripts/tidy.sh index 984ad38ec3b3..e239fc3b1a6b 100755 --- a/circuits/cpp/scripts/tidy.sh +++ b/circuits/cpp/scripts/tidy.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu # Run clang-tidy on all C++ source files # diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index 3cb814fc4d51..58aba5449eb3 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -16,4 +16,4 @@ forge install --no-commit git submodule update --init --recursive ./lib # Compile contracts -forge build \ No newline at end of file +forge build diff --git a/l1-contracts/scripts/install_foundry.sh b/l1-contracts/scripts/install_foundry.sh index 3842f6a4318b..e085beb76d59 100755 --- a/l1-contracts/scripts/install_foundry.sh +++ b/l1-contracts/scripts/install_foundry.sh @@ -18,4 +18,4 @@ chmod +x $BIN_PATH export PATH=$FOUNDRY_BIN_DIR:$PATH # Use version. -foundryup \ No newline at end of file +foundryup diff --git a/scripts/ci/store_test_benchmark_logs b/scripts/ci/store_test_benchmark_logs index 057d64b4bc33..1d9350e6c980 100755 --- a/scripts/ci/store_test_benchmark_logs +++ b/scripts/ci/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu AZTEC_GITHUB_TOKEN=$1 @@ -33,11 +33,11 @@ cat /tmp/csv/new.csv \ if [ -s /tmp/csv/trimmed.csv ]; then cd /tmp - git clone --depth 1 https://$AZTEC_GITHUB_TOKEN:@github.com/AztecProtocol/benchmark-archive + git clone --depth 1 git@github.com-logs:AztecProtocol/benchmark-archive.git cd benchmark-archive - git config --global user.name AztecBot - git config --global user.email tech@aztecprotocol.com + git config user.email "circleci@bot" + git config user.name "CircleCi Bot" cat /tmp/csv/trimmed.csv >>benchmarks.csv git add benchmarks.csv git commit -m "Added information from branch $BRANCH commit $COMMIT_HASH" diff --git a/scripts/git-subrepo/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/bashplus/bin/bash+ index a691460242d9..e0bc1f73913a 100755 --- a/scripts/git-subrepo/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -e +set -eu shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash index dccdac08376f..b52e17b55a62 100644 --- a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -e +set -eu [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ index a691460242d9..e0bc1f73913a 100755 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -e +set -eu shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash index 2e1a70c20795..e0e15cfc49d7 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -e +set -eu [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/lib/git-subrepo b/scripts/git-subrepo/lib/git-subrepo index a6d5d96a1f6a..8ee1621915c5 100755 --- a/scripts/git-subrepo/lib/git-subrepo +++ b/scripts/git-subrepo/lib/git-subrepo @@ -7,7 +7,7 @@ # shellcheck disable=1090,1091,2034 # Exit on any errors: -set -e +set -eu export FILTER_BRANCH_SQUELCH_WARNING=1 @@ -1902,7 +1902,7 @@ RUN() { fi fi rc=$? - set -e + set -eu if [[ $rc -ne 0 ]]; then OK=false diff --git a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash index 123bb54cbb5c..98e34dd272b0 100644 --- a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash +++ b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash @@ -2,7 +2,7 @@ # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -e +set -eu help:all() { cat <<'...' diff --git a/scripts/git-subrepo/note/recreate-rebase-conflict.sh b/scripts/git-subrepo/note/recreate-rebase-conflict.sh index 21bf8365b75a..8e1103e575d8 100644 --- a/scripts/git-subrepo/note/recreate-rebase-conflict.sh +++ b/scripts/git-subrepo/note/recreate-rebase-conflict.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -eu set -x # Make a directory to work in: diff --git a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl index 40b6af8f2b38..5dd949e6ed90 100644 --- a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl +++ b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl @@ -33,7 +33,7 @@ sub write_start { # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -e +set -eu ... } diff --git a/scripts/git_subrepo.sh b/scripts/git_subrepo.sh index df9743a2bd5f..dd87b304408d 100755 --- a/scripts/git_subrepo.sh +++ b/scripts/git_subrepo.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -e SCRIPT_DIR=$(dirname "$(realpath "$0")") diff --git a/scripts/update.sh b/scripts/update.sh index 76446d014a49..ab78b6827dc4 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -14,4 +14,4 @@ echo -e '\n\033[1mRebuild Noir contracts...\033[0m' (cd yarn-project/noir-contracts && yarn noir:build:all 2> /dev/null) echo -e '\n\033[1mRebuild circuits wasm...\033[0m' -(cd circuits/cpp && cmake --build --preset wasm -j --target aztec3-circuits.wasm) \ No newline at end of file +(cd circuits/cpp && cmake --build --preset wasm -j --target aztec3-circuits.wasm) diff --git a/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml b/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml index d677792416cb..a37fd7880fcd 100644 --- a/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml +++ b/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml @@ -13,7 +13,7 @@ services: - '8545:8545' sandbox: - image: $ECR_URL/aztec-sandbox:cache-$COMMIT_HASH + image: $ECR_URL/aztec-sandbox:cache-$CONTENT_HASH environment: DEBUG: 'aztec:*' ETHEREUM_HOST: http://fork:8545 diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index 9a99a2018e66..dac0f0bd921e 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -e +set -eu export TEST=$1 export IMAGE=${2:-canary} @@ -12,12 +12,11 @@ if [ "$TEST" = "uniswap_trade_on_l1_from_l2.test.ts" ]; then export FORK_BLOCK_NUMBER=17514288 fi -if [ -n "$COMMIT_HASH" ]; then - $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$COMMIT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$COMMIT_HASH aztecprotocol/canary:latest -fi +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$CONTENT_HASH" +docker pull $IMAGE_URI +docker tag $IMAGE_URI aztecprotocol/canary:latest docker-compose rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from canary \ No newline at end of file +docker-compose -f $COMPOSE_FILE up --exit-code-from canary diff --git a/yarn-project/canary/scripts/update_packages.sh b/yarn-project/canary/scripts/update_packages.sh index 7fa52399a234..0b5a3c59b43f 100755 --- a/yarn-project/canary/scripts/update_packages.sh +++ b/yarn-project/canary/scripts/update_packages.sh @@ -1,4 +1,4 @@ -set -e +set -eu COMMIT_TAG=$1 @@ -20,4 +20,4 @@ for PKG in $(jq --raw-output ".dependencies | keys[] | select(contains(\"@aztec/ jq --arg v $VERSION ".dependencies[\"$PKG\"] = \$v" package.json > $TMP && mv $TMP package.json done -jq ".references = []" tsconfig.json > $TMP && mv $TMP tsconfig.json \ No newline at end of file +jq ".references = []" tsconfig.json > $TMP && mv $TMP tsconfig.json diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index 96adcc9e8e27..e8159655bf20 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -14,7 +14,8 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Script to run and args. -set -e +set -eu +set -x REPOSITORY=$1 shift @@ -23,12 +24,11 @@ shift SCRIPT_TO_RUN=$1 shift -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $SUCCESS_TAG) - -echo "Last successful commit for $SUCCESS_TAG: $LAST_SUCCESSFUL_COMMIT" +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Content hash: $CONTENT_HASH" echo "Script to run is $SCRIPT_TO_RUN $@" -if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then - "$SCRIPT_TO_RUN" "$@" - tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$SUCCESS_TAG +if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then + CONTENT_HASH="$CONTENT_HASH" "$SCRIPT_TO_RUN" "$@" + tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG fi diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index ad3f295edf4e..fad024b1eb2d 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -1,19 +1,17 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -e +set -eu export TEST=$1 export COMPOSE_FILE=${2:-docker-compose.yml} -if [ -n "$COMMIT_HASH" ]; then - $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - for REPO in aztec.js end-to-end aztec-sandbox; do - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$COMMIT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$COMMIT_HASH aztecprotocol/$REPO:latest - done -fi +for REPO in aztec.js end-to-end aztec-sandbox; do + docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH + docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest +done docker-compose rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end \ No newline at end of file +docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 98e2b67ffbad..e77366224170 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -1,19 +1,17 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_run_script). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -e +set -eu export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} -if [ -n "$COMMIT_HASH" ]; then - aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com +aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com - for REPO in end-to-end; do - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$COMMIT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$COMMIT_HASH aztecprotocol/$REPO:latest - done -fi +for REPO in end-to-end; do + docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH + docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest +done docker-compose -f $COMPOSE_FILE rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end \ No newline at end of file +docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end diff --git a/yarn-project/end-to-end/scripts/start_e2e.sh b/yarn-project/end-to-end/scripts/start_e2e.sh index 617982ba7e80..018c3d79603a 100755 --- a/yarn-project/end-to-end/scripts/start_e2e.sh +++ b/yarn-project/end-to-end/scripts/start_e2e.sh @@ -2,4 +2,4 @@ set -eu export NODE_NO_WARNINGS=1 -node ${NODE_ARGS-} --openssl-legacy-provider --experimental-vm-modules $(yarn bin jest) --no-cache --runInBand --passWithNoTests $@ \ No newline at end of file +node ${NODE_ARGS-} --openssl-legacy-provider --experimental-vm-modules $(yarn bin jest) --no-cache --runInBand --passWithNoTests $@ diff --git a/yarn-project/end-to-end/scripts/start_p2p_e2e.sh b/yarn-project/end-to-end/scripts/start_p2p_e2e.sh index 144fda442117..9b1b661a7feb 100755 --- a/yarn-project/end-to-end/scripts/start_p2p_e2e.sh +++ b/yarn-project/end-to-end/scripts/start_p2p_e2e.sh @@ -14,4 +14,4 @@ export P2P_SERVER='false' export P2P_ENABLED='true' export DEBUG='aztec:*,libp2p:*' -yarn test e2e_p2p_network.test.ts \ No newline at end of file +yarn test e2e_p2p_network.test.ts diff --git a/yarn-project/noir-contracts/scripts/install_noir.sh b/yarn-project/noir-contracts/scripts/install_noir.sh index 126cd93c1e8f..c0c9d70d7298 100755 --- a/yarn-project/noir-contracts/scripts/install_noir.sh +++ b/yarn-project/noir-contracts/scripts/install_noir.sh @@ -5,4 +5,4 @@ set -eu VERSION="aztec" # Install nargo -noirup -v $VERSION \ No newline at end of file +noirup -v $VERSION diff --git a/yarn-project/noir-contracts/scripts/install_noirup.sh b/yarn-project/noir-contracts/scripts/install_noirup.sh index 86d056bc1858..11ba9b15d312 100755 --- a/yarn-project/noir-contracts/scripts/install_noirup.sh +++ b/yarn-project/noir-contracts/scripts/install_noirup.sh @@ -19,4 +19,4 @@ mkdir -p $NARGO_MAN_DIR curl -# -Ls $BIN_URL -o $BIN_PATH chmod +x $BIN_PATH -export PATH=$NARGO_BIN_DIR:$PATH \ No newline at end of file +export PATH=$NARGO_BIN_DIR:$PATH diff --git a/yarn-project/noir-contracts/src/scripts/compile.sh b/yarn-project/noir-contracts/src/scripts/compile.sh index 99500bb4de9e..3edb7482cadf 100755 --- a/yarn-project/noir-contracts/src/scripts/compile.sh +++ b/yarn-project/noir-contracts/src/scripts/compile.sh @@ -13,4 +13,4 @@ set -euo pipefail; # Run build scripts ./scripts/compile.sh "$@" -./scripts/types.sh "$@" \ No newline at end of file +./scripts/types.sh "$@" diff --git a/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh b/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh index 613d11259719..2abace6942da 100755 --- a/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh +++ b/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh @@ -5,4 +5,4 @@ cd .. export P2P_TCP_LISTEN_PORT=40400 export PEER_ID='0a260024080112205ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e1224080112205ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e1a44080112402df8b977f356c6e34fa021c9647973234dff4df706c185794405aafb556723cf5ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e' yarn build -yarn start \ No newline at end of file +yarn start diff --git a/yarn-project/types/Dockerfile b/yarn-project/types/Dockerfile index 6031d076e0c3..3f97c9648b02 100644 --- a/yarn-project/types/Dockerfile +++ b/yarn-project/types/Dockerfile @@ -11,4 +11,4 @@ RUN yarn workspaces focus --production > /dev/null FROM node:18-alpine COPY --from=builder /usr/src/yarn-project/types /usr/src/yarn-project/types WORKDIR /usr/src/yarn-project/types -ENTRYPOINT ["yarn"] \ No newline at end of file +ENTRYPOINT ["yarn"] diff --git a/yarn-project/yarn-project-base/scripts/update_build_manifest.mjs b/yarn-project/yarn-project-base/scripts/update_build_manifest.mjs index 36adc602a58b..b915dd8733d1 100644 --- a/yarn-project/yarn-project-base/scripts/update_build_manifest.mjs +++ b/yarn-project/yarn-project-base/scripts/update_build_manifest.mjs @@ -21,7 +21,10 @@ function updateBuildManifest(buildManifestFile, allDependencies, projectKey, opt // Update the "dependencies" key in the corresponding section of the buildManifestData // Take just the folder name component - const updatedDependencies = aztecDependencies.map(packageName => packageName.split('/')[1]); + // Filter out dependencies that are not themselves in the manifest: + const updatedDependencies = aztecDependencies + .map(packageName => packageName.split('/')[1]) + .filter(depProjectKey => !!buildManifestData[depProjectKey]); // If we are just checking, throw if dependencies don't match if (options.checkOnly) {