From 21cb22bbb2a0b894a8609f835a8ad28da44a52bd Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 11:34:21 -0400 Subject: [PATCH 01/17] git subrepo pull --force circuits/cpp/barretenberg subrepo: subdir: "circuits/cpp/barretenberg" merged: "c6c469aa5d" upstream: origin: "https://github.com/AztecProtocol/barretenberg" branch: "master" commit: "c6c469aa5d" git-subrepo: version: "0.4.6" origin: "???" commit: "???" --- .gitmodules | 12 ------------ circuits/cpp/barretenberg/.gitrepo | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.gitmodules b/.gitmodules index 64ffb023e53..24ef9571bd4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,18 +1,6 @@ -[submodule "legacy-barretenberg-build-system"] - path = circuits/cpp/barretenberg/build-system - url = https://github.com/AztecProtocol/build-system [submodule "l1-contracts/lib/openzeppelin-contracts"] path = l1-contracts/lib/openzeppelin-contracts url = https://github.com/openzeppelin/openzeppelin-contracts [submodule "l1-contracts/lib/forge-std"] path = l1-contracts/lib/forge-std url = https://github.com/foundry-rs/forge-std -[submodule "circuits/cpp/barretenberg/sol/lib/forge-std"] - path = circuits/cpp/barretenberg/sol/lib/forge-std - url = https://github.com/foundry-rs/forge-std -[submodule "circuits/cpp/barretenberg/sol/lib/solidity-stringutils"] - path = circuits/cpp/barretenberg/sol/lib/solidity-stringutils - url = https://github.com/Arachnid/solidity-stringutils -[submodule "circuits/cpp/barretenberg/sol/lib/openzeppelin-contracts"] - path = circuits/cpp/barretenberg/sol/lib/openzeppelin-contracts - url = https://github.com/OpenZeppelin/openzeppelin-contracts diff --git a/circuits/cpp/barretenberg/.gitrepo b/circuits/cpp/barretenberg/.gitrepo index aa21fe09a25..114164437d3 100644 --- a/circuits/cpp/barretenberg/.gitrepo +++ b/circuits/cpp/barretenberg/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/AztecProtocol/barretenberg branch = master - commit = 155e32ba1498bb18ebbd86d784aea8b56f910f8f - parent = 74c83c5e1436f391eef435926c2da1d508d67713 + commit = c6c469aa5da7c6973f656ddf8af4fb20c3e8e4f6 + parent = 0a2a515ecf52849cce1e45a7b39f44d420b43f34 method = merge cmdver = 0.4.6 From 10772d3ace43d01e4a45495baccd40d0ade20ee6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 11:36:04 -0400 Subject: [PATCH 02/17] Fix bb --- .gitmodules | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitmodules b/.gitmodules index 24ef9571bd4..64ffb023e53 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,18 @@ +[submodule "legacy-barretenberg-build-system"] + path = circuits/cpp/barretenberg/build-system + url = https://github.com/AztecProtocol/build-system [submodule "l1-contracts/lib/openzeppelin-contracts"] path = l1-contracts/lib/openzeppelin-contracts url = https://github.com/openzeppelin/openzeppelin-contracts [submodule "l1-contracts/lib/forge-std"] path = l1-contracts/lib/forge-std url = https://github.com/foundry-rs/forge-std +[submodule "circuits/cpp/barretenberg/sol/lib/forge-std"] + path = circuits/cpp/barretenberg/sol/lib/forge-std + url = https://github.com/foundry-rs/forge-std +[submodule "circuits/cpp/barretenberg/sol/lib/solidity-stringutils"] + path = circuits/cpp/barretenberg/sol/lib/solidity-stringutils + url = https://github.com/Arachnid/solidity-stringutils +[submodule "circuits/cpp/barretenberg/sol/lib/openzeppelin-contracts"] + path = circuits/cpp/barretenberg/sol/lib/openzeppelin-contracts + url = https://github.com/OpenZeppelin/openzeppelin-contracts From b00790b3f3260b789ab1b0929cee138ea15d781e Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 11:44:51 -0400 Subject: [PATCH 03/17] fix master --- yarn-project/end-to-end/scripts/run_tests_local | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index e7736622417..1ea02e7bd9b 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -8,9 +8,11 @@ export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com +BS=$(git rev-parse --show-toplevel)/build-system/scripts + for REPO in end-to-end; do - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest + docker pull $($BS/calculate_image_uri $REPO) + docker tag $($BS/calculate_image_uri $REPO) aztecprotocol/$REPO:latest done docker-compose -f $COMPOSE_FILE rm -f From cbe0b56c175e510b32bb6c3e8012cddb39b7e96b Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 11:58:07 -0400 Subject: [PATCH 04/17] fix e2e sandbox test --- yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml | 2 +- yarn-project/canary/scripts/run_tests | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml b/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml index a37fd7880fc..bd97f00d823 100644 --- a/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml +++ b/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml @@ -13,7 +13,7 @@ services: - '8545:8545' sandbox: - image: $ECR_URL/aztec-sandbox:cache-$CONTENT_HASH + image: aztecprotocol/sandbox:latest environment: DEBUG: 'aztec:*' ETHEREUM_HOST: http://fork:8545 diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index 44c0b2e9dca..e2fd492f287 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -18,5 +18,9 @@ IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_imag docker pull $IMAGE_URI docker tag $IMAGE_URI aztecprotocol/canary:latest +SANDBOX_IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri aztec-sandbox) +docker pull $SANDBOX_IMAGE_URI +docker tag $SANDBOX_IMAGE_URI aztecprotocol/sandbox:latest + docker-compose rm -f docker-compose -f $COMPOSE_FILE up --exit-code-from canary From 470c791b5ca06b5b91f6e327960c4b86028fdf20 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 15:00:35 -0400 Subject: [PATCH 05/17] Add sooner logins --- build-system/scripts/build | 6 ++++-- yarn-project/canary/scripts/run_tests | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index 2f59649e273..4811a2d332e 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -35,6 +35,9 @@ echo "Working directory: $PWD" echo "Dockerfile: $DOCKERFILE" echo "Build directory: $BUILD_DIR" +# We login to allow more docker +echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin + function fetch_image() { echo "Pulling: $1" if ! docker pull $1 > /dev/null 2>&1; then @@ -77,8 +80,7 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then popd fi -# Pull latest parents that are not ours. We also do not want to pull images suffixed by _, this is how we scope intermediate build images. -echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin +# Pull latest parents that are not ours. We also do not want to pull images suffixed by _, this is how we scope intermediate build images. login -u aztecprotocolci --password-stdin PARENTS=$(cat $DOCKERFILE | sed -n -e 's/^FROM \([^[:space:]]\+\).*/\1/p' | sed '/_$/d' | grep -v $ECR_DEPLOY_URL | sort | uniq) for PARENT in $PARENTS; do fetch_image $PARENT diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index e2fd492f287..82182d33ad8 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -14,6 +14,8 @@ fi $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com + IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $IMAGE) docker pull $IMAGE_URI docker tag $IMAGE_URI aztecprotocol/canary:latest From b168b1f04068fc8070e2fb54f305023ed2621906 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 15:04:27 -0400 Subject: [PATCH 06/17] Rebuild patterns. Try it all and see if we hit intermittent issues --- build-system/scripts/check_rebuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index d80e9493684..f12d4a22573 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -19,5 +19,5 @@ elif image_exists $REPOSITORY tainted; then exit 1 else echo "No rebuild required." - exit 0 + exit 1 fi From 7a2c5ac09bc04fb5e0f72c3f16bf94331084f6c5 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 15:17:55 -0400 Subject: [PATCH 07/17] Revert check_rebuild. Add comment --- build-system/scripts/build | 5 +++-- build-system/scripts/check_rebuild | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index 4811a2d332e..42daf74adc4 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -35,7 +35,8 @@ echo "Working directory: $PWD" echo "Dockerfile: $DOCKERFILE" echo "Build directory: $BUILD_DIR" -# We login to allow more docker +# Important! While we can pull tags without logging in, we risk getting throttled and CI breaking. +# We should always login first. echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin function fetch_image() { @@ -80,7 +81,7 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then popd fi -# Pull latest parents that are not ours. We also do not want to pull images suffixed by _, this is how we scope intermediate build images. login -u aztecprotocolci --password-stdin +# Pull latest parents that are not ours. We also do not want to pull images suffixed by _, this is how we scope intermediate build images. PARENTS=$(cat $DOCKERFILE | sed -n -e 's/^FROM \([^[:space:]]\+\).*/\1/p' | sed '/_$/d' | grep -v $ECR_DEPLOY_URL | sort | uniq) for PARENT in $PARENTS; do fetch_image $PARENT diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index f12d4a22573..d80e9493684 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -19,5 +19,5 @@ elif image_exists $REPOSITORY tainted; then exit 1 else echo "No rebuild required." - exit 1 + exit 0 fi From 74ef7eede330fd41576fe59ae897ed80685e846b Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 15:20:51 -0400 Subject: [PATCH 08/17] Try C++ change. --- .../cpp/src/barretenberg/honk/flavor/flavor.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuits/cpp/barretenberg/cpp/src/barretenberg/honk/flavor/flavor.test.cpp b/circuits/cpp/barretenberg/cpp/src/barretenberg/honk/flavor/flavor.test.cpp index 6a227e33f45..a7d749f5b8e 100644 --- a/circuits/cpp/barretenberg/cpp/src/barretenberg/honk/flavor/flavor.test.cpp +++ b/circuits/cpp/barretenberg/cpp/src/barretenberg/honk/flavor/flavor.test.cpp @@ -14,7 +14,7 @@ TEST(Flavor, StandardGetters) using FF = Flavor::FF; using ProvingKey = typename Flavor::ProvingKey; - ProvingKey proving_key = []() { return Flavor::ProvingKey(/*circuit_size=*/4, /*num_public_inputs=*/0); }(); + ProvingKey proving_key = []() { return Flavor::ProvingKey(/*circuit_size =*/4, /*num_public_inputs=*/0); }(); // set size_t coset_idx = 0; From aaa0a0bd991cda848e7c6386b377055063d278e9 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 22:30:24 -0400 Subject: [PATCH 09/17] Consolidate workflows in mirror --- .github/workflows/mirror_repos.yml | 80 ++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/mirror_repos.yml diff --git a/.github/workflows/mirror_repos.yml b/.github/workflows/mirror_repos.yml new file mode 100644 index 00000000000..94e73836788 --- /dev/null +++ b/.github/workflows/mirror_repos.yml @@ -0,0 +1,80 @@ +# We push using git subrepo (https://github.com/ingydotnet/git-subrepo) +# with some logic to recover from squashed parent commits +# We first identify ourselves, needed to commit. +# Then push to subrepo, commit to master. The commit is needed +# to continue to replay. If we still hit issues such as this +# action failing due to upstream changes, a manual resolution +# PR with ./scripts/git_subrepo.sh pull will be needed. +name: Mirror Repositories + +concurrency: + group: mirror-repositories +on: + push: + branches: + - master + paths: + - "docs/**" + - "!docs/.gitrepo" + - "build-system/**" + - "!build-system/.gitrepo" + - "circuits/cpp/barretenberg/**" + - "!circuits/cpp/barretenberg/.gitrepo" + +jobs: + mirror-to-docs-repo: + runs-on: ubuntu-latest + if: "contains(github.event.head_commit.modified, 'docs/')" + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + - name: Push to docs repo + run: | + SUBREPO_PATH=docs + git config --global user.name AztecBot + git config --global user.email tech@aztecprotocol.com + ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=main + git fetch # in case a commit came after this + git rebase origin/master + git push + + mirror-to-build-system-repo: + runs-on: ubuntu-latest + if: "contains(github.event.head_commit.modified, 'build-system/')" + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + - name: Push to build-system repo + run: | + SUBREPO_PATH=build-system + git config --global user.name AztecBot + git config --global user.email tech@aztecprotocol.com + ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=maste + git fetch # in case a commit came after this + git rebase origin/masterr + git push + + mirror-to-barretenberg-repo: + runs-on: ubuntu-latest + if: "contains(github.event.head_commit.modified, 'circuits/cpp/barretenberg/')" + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + - name: Push to barretenberg repo + run: | + SUBREPO_PATH=circuits/cpp/barretenberg + git config --global user.name AztecBot + git config --global user.email tech@aztecprotocol.com + ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=master + git fetch # in case a commit came after this + git rebase origin/master + git push From 7cedc5d658f71272eec2201e517ef1e12de72718 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 22:31:05 -0400 Subject: [PATCH 10/17] Revert --- build-system/.gitrepo | 4 ++-- build-system/scripts/build | 5 +---- build-system/scripts/build_local | 13 +++++++++++-- build-system/scripts/setup_env | 3 ++- .../src/barretenberg/honk/flavor/flavor.test.cpp | 2 +- yarn-project/canary/scripts/run_tests | 2 -- 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/build-system/.gitrepo b/build-system/.gitrepo index 97c4e7c1839..0b7033f2c5d 100644 --- a/build-system/.gitrepo +++ b/build-system/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/AztecProtocol/build-system branch = master - commit = fcb406e61bf7a401b64140d62ad4e684904bff9f - parent = 0a2a515ecf52849cce1e45a7b39f44d420b43f34 + commit = 36a9071caf2d96d9685396611b85726e8d912afe + parent = 1aa6f5934cd97dd32d81e490013f8ef7d1e14ec7 method = merge cmdver = 0.4.6 diff --git a/build-system/scripts/build b/build-system/scripts/build index 42daf74adc4..2f59649e273 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -35,10 +35,6 @@ echo "Working directory: $PWD" echo "Dockerfile: $DOCKERFILE" echo "Build directory: $BUILD_DIR" -# Important! While we can pull tags without logging in, we risk getting throttled and CI breaking. -# We should always login first. -echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin - function fetch_image() { echo "Pulling: $1" if ! docker pull $1 > /dev/null 2>&1; then @@ -82,6 +78,7 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then fi # Pull latest parents that are not ours. We also do not want to pull images suffixed by _, this is how we scope intermediate build images. +echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin PARENTS=$(cat $DOCKERFILE | sed -n -e 's/^FROM \([^[:space:]]\+\).*/\1/p' | sed '/_$/d' | grep -v $ECR_DEPLOY_URL | sort | uniq) for PARENT in $PARENTS; do fetch_image $PARENT diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index bb1d6340844..12e2f0c97cc 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -9,7 +9,11 @@ set -eu TARGET_PROJECT=$1 -ONLY_TARGET=$2 +ONLY_TARGET=${2:-} + +if [ -n "${NO_CACHE:-}" ]; then + ADDITIONAL_ARGS=--no-cache +fi ROOT=$(git rev-parse --show-toplevel) source $ROOT/build_manifest.sh @@ -20,6 +24,7 @@ for E in ${PROJECTS[@]}; do WORKING_DIR=${ARR[1]} DOCKERFILE=${ARR[2]} REPO=${ARR[3]:-$PROJECT_DIR_NAME} + LAUNCH=${ARR[4]:-} if [ "$ONLY_TARGET" == "true" -a ! "$PROJECT_DIR_NAME" = "$TARGET_PROJECT" ]; then continue @@ -44,9 +49,13 @@ for E in ${PROJECTS[@]}; do echo "*** Building $PROJECT_DIR_NAME:$DOCKERFILE -> $REPO ***" echo - time docker build --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $ECR_DEPLOY_URL/$REPO:latest . + time docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $ECR_DEPLOY_URL/$REPO:latest . docker tag $ECR_DEPLOY_URL/$REPO:latest aztecprotocol/$REPO:latest + if [ -n "$LAUNCH" ]; then + docker run -ti --rm aztecprotocol/$REPO:latest + fi + if [ "$PROJECT_DIR_NAME" = "$TARGET_PROJECT" ]; then break fi diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 2df97fbfd5a..633e82881e4 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -14,6 +14,7 @@ JOB_NAME=$3 GIT_REPOSITORY_URL=${4:-} BRANCH=${5:-} +BASH_ENV=${BASH_ENV:-} BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) PROJECT=$(cat PROJECT) VERSION=$(cat VERSION) @@ -95,7 +96,7 @@ source $BASH_ENV if [ -n "$COMMIT_HASH" ]; then # Install and ensure correct permissions on build instance key. mkdir -p ~/.ssh - echo $BUILD_INSTANCE_KEY | base64 -d > ~/.ssh/build_instance_key + echo ${BUILD_INSTANCE_KEY:-} | base64 -d > ~/.ssh/build_instance_key chmod 600 ~/.ssh/build_instance_key fi diff --git a/circuits/cpp/barretenberg/cpp/src/barretenberg/honk/flavor/flavor.test.cpp b/circuits/cpp/barretenberg/cpp/src/barretenberg/honk/flavor/flavor.test.cpp index a7d749f5b8e..6a227e33f45 100644 --- a/circuits/cpp/barretenberg/cpp/src/barretenberg/honk/flavor/flavor.test.cpp +++ b/circuits/cpp/barretenberg/cpp/src/barretenberg/honk/flavor/flavor.test.cpp @@ -14,7 +14,7 @@ TEST(Flavor, StandardGetters) using FF = Flavor::FF; using ProvingKey = typename Flavor::ProvingKey; - ProvingKey proving_key = []() { return Flavor::ProvingKey(/*circuit_size =*/4, /*num_public_inputs=*/0); }(); + ProvingKey proving_key = []() { return Flavor::ProvingKey(/*circuit_size=*/4, /*num_public_inputs=*/0); }(); // set size_t coset_idx = 0; diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index 82182d33ad8..e2fd492f287 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -14,8 +14,6 @@ fi $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com - IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $IMAGE) docker pull $IMAGE_URI docker tag $IMAGE_URI aztecprotocol/canary:latest From 33ad7bbaac5c7ebfca346c58b42ecde0d67a12b0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 22:32:36 -0400 Subject: [PATCH 11/17] Delete other workflows --- .../workflows/mirror_barretenberg_repo.yml | 35 ------------------- .../workflows/mirror_build_system_repo.yml | 35 ------------------- .github/workflows/mirror_docs_repo.yml | 35 ------------------- 3 files changed, 105 deletions(-) delete mode 100644 .github/workflows/mirror_barretenberg_repo.yml delete mode 100644 .github/workflows/mirror_build_system_repo.yml delete mode 100644 .github/workflows/mirror_docs_repo.yml diff --git a/.github/workflows/mirror_barretenberg_repo.yml b/.github/workflows/mirror_barretenberg_repo.yml deleted file mode 100644 index b83c95e0f2a..00000000000 --- a/.github/workflows/mirror_barretenberg_repo.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Mirror to barretenberg repo - -on: - push: - branches: - - master - paths: - - 'circuits/cpp/barretenberg/**' - - '!circuits/cpp/barretenberg/.gitrepo' - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - - - name: Push to branch - run: | - # we push using git subrepo (https://github.com/ingydotnet/git-subrepo) - # with some logic to recover from squashed parent commits - SUBREPO_PATH=circuits/cpp/barretenberg - # identify ourselves, needed to commit - git config --global user.name AztecBot - git config --global user.email tech@aztecprotocol.com - # push to subrepo, commit to master. The commit is needed - # to continue to replay. If we still hit issues such as this - # action failing due to upstream changes, a manual resolution - # PR with ./scripts/git_subrepo.sh pull will be needed. - ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=master - git push # update .gitrepo on master diff --git a/.github/workflows/mirror_build_system_repo.yml b/.github/workflows/mirror_build_system_repo.yml deleted file mode 100644 index 9e85d3600f0..00000000000 --- a/.github/workflows/mirror_build_system_repo.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Mirror to build-system repo - -on: - push: - branches: - - master - paths: - - "build-system/**" - - "!build-system/.gitrepo" - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - - - name: Push to branch - run: | - # we push using git subrepo (https://github.com/ingydotnet/git-subrepo) - # with some logic to recover from squashed parent commits - SUBREPO_PATH=build-system - # identify ourselves, needed to commit - git config --global user.name AztecBot - git config --global user.email tech@aztecprotocol.com - # push to subrepo, commit to master. The commit is needed - # to continue to replay. If we still hit issues such as this - # action failing due to upstream changes, a manual resolution - # PR with ./scripts/git_subrepo.sh pull will be needed. - ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=master - git push # update .gitrepo on master diff --git a/.github/workflows/mirror_docs_repo.yml b/.github/workflows/mirror_docs_repo.yml deleted file mode 100644 index 7965b5c867d..00000000000 --- a/.github/workflows/mirror_docs_repo.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Mirror to docs repo - -on: - push: - branches: - - master - paths: - - 'docs/**' - - '!docs/.gitrepo' - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - - - name: Push to branch - run: | - # we push using git subrepo (https://github.com/ingydotnet/git-subrepo) - # with some logic to recover from squashed parent commits - SUBREPO_PATH=docs - # identify ourselves, needed to commit - git config --global user.name AztecBot - git config --global user.email tech@aztecprotocol.com - # push to subrepo, commit to master. The commit is needed - # to continue to replay. If we still hit issues such as this - # action failing due to upstream changes, a manual resolution - # PR with ./scripts/git_subrepo.sh pull will be needed. - ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=main - git push # update .gitrepo on master From cd747c214a9eb51eafc8d9167d06eb6b9e5c53cc Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 22:34:16 -0400 Subject: [PATCH 12/17] Consolidate workflows --- .github/workflows/mirror_repos.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/mirror_repos.yml b/.github/workflows/mirror_repos.yml index 94e73836788..863f15b905f 100644 --- a/.github/workflows/mirror_repos.yml +++ b/.github/workflows/mirror_repos.yml @@ -39,6 +39,7 @@ jobs: ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=main git fetch # in case a commit came after this git rebase origin/master + git commit --amend -m "$(git log -1 --pretty=%B) [skip ci]" git push mirror-to-build-system-repo: @@ -58,6 +59,7 @@ jobs: ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=maste git fetch # in case a commit came after this git rebase origin/masterr + git commit --amend -m "$(git log -1 --pretty=%B) [skip ci]" git push mirror-to-barretenberg-repo: @@ -77,4 +79,5 @@ jobs: ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=master git fetch # in case a commit came after this git rebase origin/master + git commit --amend -m "$(git log -1 --pretty=%B) [skip ci]" git push From 8dec50819ba98baf56bd598d85c8c7fa10aff3f0 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:13:35 +0000 Subject: [PATCH 13/17] feat: update script to run at night time utc (only if the repo has changes) --- .github/workflows/mirror_repos.yml | 40 ++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/.github/workflows/mirror_repos.yml b/.github/workflows/mirror_repos.yml index 863f15b905f..db12988a57b 100644 --- a/.github/workflows/mirror_repos.yml +++ b/.github/workflows/mirror_repos.yml @@ -10,16 +10,9 @@ name: Mirror Repositories concurrency: group: mirror-repositories on: - push: - branches: - - master - paths: - - "docs/**" - - "!docs/.gitrepo" - - "build-system/**" - - "!build-system/.gitrepo" - - "circuits/cpp/barretenberg/**" - - "!circuits/cpp/barretenberg/.gitrepo" + schedule: + # Run the workflow every night at 2:00 AM UTC. + cron: '0 2 * * *' jobs: mirror-to-docs-repo: @@ -31,7 +24,16 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + - name: Check for docs changes + id: changes + run: | + if [[ -n "$(git diff --name-only origin/master -- 'docs/')" ]]; then + echo "::set-output name=changes::true" + else + echo "::set-output name=changes::false" + fi - name: Push to docs repo + if: steps.changes.outputs.changes == 'true' run: | SUBREPO_PATH=docs git config --global user.name AztecBot @@ -51,7 +53,16 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + - name: Check for build system changes + id: changes + run: | + if [[ -n "$(git diff --name-only origin/master -- 'build-system/')" ]]; then + echo "::set-output name=changes::true" + else + echo "::set-output name=changes::false" + fi - name: Push to build-system repo + if: steps.changes.outputs.changes == 'true' run: | SUBREPO_PATH=build-system git config --global user.name AztecBot @@ -71,7 +82,16 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + - name: Check for barretenberg changes + id: changes + run: | + if [[ -n "$(git diff --name-only origin/master -- 'circuits/cpp/barretenberg/')" ]]; then + echo "::set-output name=changes::true" + else + echo "::set-output name=changes::false" + fi - name: Push to barretenberg repo + if: steps.changes.outputs.changes == 'true' run: | SUBREPO_PATH=circuits/cpp/barretenberg git config --global user.name AztecBot From abcde1694042893a193b6aa80b11fd391653b0a6 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:17:37 +0000 Subject: [PATCH 14/17] fix: remove verbose change check, it was already implied --- .github/workflows/mirror_repos.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/mirror_repos.yml b/.github/workflows/mirror_repos.yml index db12988a57b..a98394ad71b 100644 --- a/.github/workflows/mirror_repos.yml +++ b/.github/workflows/mirror_repos.yml @@ -24,16 +24,7 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - - name: Check for docs changes - id: changes - run: | - if [[ -n "$(git diff --name-only origin/master -- 'docs/')" ]]; then - echo "::set-output name=changes::true" - else - echo "::set-output name=changes::false" - fi - name: Push to docs repo - if: steps.changes.outputs.changes == 'true' run: | SUBREPO_PATH=docs git config --global user.name AztecBot @@ -53,16 +44,7 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - - name: Check for build system changes - id: changes - run: | - if [[ -n "$(git diff --name-only origin/master -- 'build-system/')" ]]; then - echo "::set-output name=changes::true" - else - echo "::set-output name=changes::false" - fi - name: Push to build-system repo - if: steps.changes.outputs.changes == 'true' run: | SUBREPO_PATH=build-system git config --global user.name AztecBot @@ -82,16 +64,7 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - - name: Check for barretenberg changes - id: changes - run: | - if [[ -n "$(git diff --name-only origin/master -- 'circuits/cpp/barretenberg/')" ]]; then - echo "::set-output name=changes::true" - else - echo "::set-output name=changes::false" - fi - name: Push to barretenberg repo - if: steps.changes.outputs.changes == 'true' run: | SUBREPO_PATH=circuits/cpp/barretenberg git config --global user.name AztecBot From 731570445a70a61500d8ad22b616ab632638797b Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:23:51 +0000 Subject: [PATCH 15/17] fix: update bb path --- .github/workflows/mirror_repos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mirror_repos.yml b/.github/workflows/mirror_repos.yml index a98394ad71b..2229f7c2392 100644 --- a/.github/workflows/mirror_repos.yml +++ b/.github/workflows/mirror_repos.yml @@ -57,7 +57,7 @@ jobs: mirror-to-barretenberg-repo: runs-on: ubuntu-latest - if: "contains(github.event.head_commit.modified, 'circuits/cpp/barretenberg/')" + if: "contains(github.event.head_commit.modified, 'barretenberg/')" steps: - name: Checkout uses: actions/checkout@v3 @@ -66,7 +66,7 @@ jobs: token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - name: Push to barretenberg repo run: | - SUBREPO_PATH=circuits/cpp/barretenberg + SUBREPO_PATH=barretenberg git config --global user.name AztecBot git config --global user.email tech@aztecprotocol.com ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=master From 0266660dcaab24685242d2e37448aeff6072b941 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:25:34 +0000 Subject: [PATCH 16/17] fix: merge issue --- circuits/cpp/barretenberg | 1 + 1 file changed, 1 insertion(+) create mode 120000 circuits/cpp/barretenberg diff --git a/circuits/cpp/barretenberg b/circuits/cpp/barretenberg new file mode 120000 index 00000000000..e9b54f1df3e --- /dev/null +++ b/circuits/cpp/barretenberg @@ -0,0 +1 @@ +../../barretenberg \ No newline at end of file From 83ba9f4e1676e2103d21113b37247444e61c6fb5 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:43:43 +0000 Subject: [PATCH 17/17] fix: adams suggestions --- .github/workflows/mirror_repos.yml | 39 ++++++++++++++++-------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/mirror_repos.yml b/.github/workflows/mirror_repos.yml index 2229f7c2392..c3cec86a551 100644 --- a/.github/workflows/mirror_repos.yml +++ b/.github/workflows/mirror_repos.yml @@ -17,7 +17,6 @@ on: jobs: mirror-to-docs-repo: runs-on: ubuntu-latest - if: "contains(github.event.head_commit.modified, 'docs/')" steps: - name: Checkout uses: actions/checkout@v3 @@ -29,15 +28,16 @@ jobs: SUBREPO_PATH=docs git config --global user.name AztecBot git config --global user.email tech@aztecprotocol.com - ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=main - git fetch # in case a commit came after this - git rebase origin/master - git commit --amend -m "$(git log -1 --pretty=%B) [skip ci]" - git push + + if ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=main; then + git fetch # in case a commit came after this + git rebase origin/master + git commit --amend -m "$(git log -1 --pretty=%B) [skip ci]" + git push + fi mirror-to-build-system-repo: runs-on: ubuntu-latest - if: "contains(github.event.head_commit.modified, 'build-system/')" steps: - name: Checkout uses: actions/checkout@v3 @@ -49,15 +49,16 @@ jobs: SUBREPO_PATH=build-system git config --global user.name AztecBot git config --global user.email tech@aztecprotocol.com - ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=maste - git fetch # in case a commit came after this - git rebase origin/masterr - git commit --amend -m "$(git log -1 --pretty=%B) [skip ci]" - git push + + if ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=main; then + git fetch # in case a commit came after this + git rebase origin/master + git commit --amend -m "$(git log -1 --pretty=%B) [skip ci]" + git push + fi mirror-to-barretenberg-repo: runs-on: ubuntu-latest - if: "contains(github.event.head_commit.modified, 'barretenberg/')" steps: - name: Checkout uses: actions/checkout@v3 @@ -69,8 +70,10 @@ jobs: SUBREPO_PATH=barretenberg git config --global user.name AztecBot git config --global user.email tech@aztecprotocol.com - ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=master - git fetch # in case a commit came after this - git rebase origin/master - git commit --amend -m "$(git log -1 --pretty=%B) [skip ci]" - git push + + if ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=main; then + git fetch # in case a commit came after this + git rebase origin/master + git commit --amend -m "$(git log -1 --pretty=%B) [skip ci]" + git push + fi