From 9c19912a1b5eb531d5645e2324861be09328feb7 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Wed, 29 Nov 2023 23:45:41 +0000 Subject: [PATCH 01/37] wip --- .circleci/config.yml | 26 +++++++++++++++---- build_manifest.yml | 14 ++++++++--- yarn-project/Dockerfile.prod | 18 +++++++++++++ yarn-project/aztec-faucet/Dockerfile | 14 ++--------- yarn-project/aztec-sandbox/Dockerfile | 32 +++--------------------- yarn-project/aztec-sandbox/package.json | 2 +- yarn-project/cli/Dockerfile | 27 +++++--------------- yarn-project/cli/package.json | 2 +- yarn-project/end-to-end/Dockerfile | 2 +- yarn-project/scripts/version_packages.sh | 8 ++++++ 10 files changed, 72 insertions(+), 73 deletions(-) create mode 100644 yarn-project/Dockerfile.prod create mode 100755 yarn-project/scripts/version_packages.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 06d72664280..1b9ef6007d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,6 +346,17 @@ jobs: name: Build command: build yarn-project | add_timestamps + yarn-project-prod: + machine: + image: ubuntu-2204:2023.07.2 + resource_class: large + steps: + - *checkout + - *setup_env + - run: + name: Build + command: build yarn-project-prod | add_timestamps + yarn-project-formatting: machine: image: ubuntu-2204:2023.07.2 @@ -985,6 +996,11 @@ defaults_yarn_project: &defaults_yarn_project - yarn-project <<: *defaults +defaults_yarn_project_prod: &defaults_yarn_project_prod + requires: + - yarn-project-prod + <<: *defaults + defaults_deploy: &defaults_deploy requires: - end @@ -1073,11 +1089,11 @@ workflows: - yarn-project-tests: *defaults_yarn_project - end-to-end: *defaults_yarn_project - build-docs: *defaults_yarn_project - - aztec-sandbox-x86_64: *defaults_yarn_project - - aztec-sandbox-arm64: *defaults_yarn_project - - cli-x86_64: *defaults_yarn_project - - cli-arm64: *defaults_yarn_project - - aztec-faucet: *defaults_yarn_project + - aztec-sandbox-x86_64: *defaults_yarn_project_prod + - aztec-sandbox-arm64: *defaults_yarn_project_prod + - cli-x86_64: *defaults_yarn_project_prod + - cli-arm64: *defaults_yarn_project_prod + - aztec-faucet: *defaults_yarn_project_prod - ecr-manifest: requires: - aztec-sandbox-x86_64 diff --git a/build_manifest.yml b/build_manifest.yml index 09c96546ecb..337992d1786 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -96,23 +96,31 @@ yarn-project: dependencies: - yarn-project-base +yarn-project-prod: + buildDir: yarn-project + dockerfile: Dockerfile.prod + rebuildPatterns: + - ^yarn-project/Dockerfile.prod + dependencies: + - yarn-project + aztec-sandbox: buildDir: yarn-project projectDir: yarn-project/aztec-sandbox dependencies: - - yarn-project + - yarn-project-prod aztec-faucet: buildDir: yarn-project projectDir: yarn-project/aztec-faucet dependencies: - - yarn-project + - yarn-project-prod cli: buildDir: yarn-project projectDir: yarn-project/cli dependencies: - - yarn-project + - yarn-project-prod boxes-blank-react: buildDir: yarn-project diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod new file mode 100644 index 00000000000..e9c9bbaef63 --- /dev/null +++ b/yarn-project/Dockerfile.prod @@ -0,0 +1,18 @@ +# This productionifies the workspace, removing all developer dependencies and producing a final slim image from which +# we then generate downstream multiarch containers to execute the specific projects. +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project + +ARG COMMIT_TAG="" +RUN ./scripts/version_packages.sh +# Productionify. See comment in yarn-project-base/Dockerfile. +RUN yarn workspaces focus @aztec/cli @aztec/aztec-sandbox @aztec/aztec-faucet --production && \ + yarn cache clean && \ + rm -rf ./**/src + +# Create fresh minimal size image. +FROM node:18-alpine +COPY --from=yarn-project /usr/src/yarn-project /usr/src/yarn-project +COPY --from=yarn-project /usr/src/barretenberg/ts/package /usr/src/barretenberg/ts/package +COPY --from=yarn-project /usr/src/noir/packages /usr/src/noir/packages +# Just until weird source-resolver bug fixed. +COPY --from=yarn-project /usr/src/noir/compiler /usr/src/noir/compiler \ No newline at end of file diff --git a/yarn-project/aztec-faucet/Dockerfile b/yarn-project/aztec-faucet/Dockerfile index d524e04c8ef..be0711dbb16 100644 --- a/yarn-project/aztec-faucet/Dockerfile +++ b/yarn-project/aztec-faucet/Dockerfile @@ -1,14 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder - +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod WORKDIR /usr/src/yarn-project/aztec-faucet - -# Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn cache clean && yarn workspaces focus --production - -# Create final, minimal size image. -FROM node:18-alpine -COPY --from=builder /usr/src/ /usr/src/ -WORKDIR /usr/src/yarn-project/aztec-faucet -ENTRYPOINT ["yarn"] -CMD [ "start" ] +ENTRYPOINT ["yarn", "start"] EXPOSE 8080 diff --git a/yarn-project/aztec-sandbox/Dockerfile b/yarn-project/aztec-sandbox/Dockerfile index 4dfc92a9596..810a1e23e83 100644 --- a/yarn-project/aztec-sandbox/Dockerfile +++ b/yarn-project/aztec-sandbox/Dockerfile @@ -1,34 +1,8 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod # Need new arch specific image. -FROM node:18-alpine as builder -COPY --from=yarn-project /usr/src /usr/src -ARG COMMIT_TAG="" - -# Update pxe version if COMMIT_TAG has been used. -WORKDIR /usr/src/yarn-project/pxe -RUN if [[ -n "${COMMIT_TAG}" ]]; then \ - jq --arg v ${COMMIT_TAG} '.version = $v' package.json > _temp && mv _temp package.json; \ - fi - -# Update sandbox version if COMMIT_TAG has been used. -WORKDIR /usr/src/yarn-project/aztec-sandbox -RUN if [[ -n "${COMMIT_TAG}" ]]; then \ - jq --arg v ${COMMIT_TAG} '.version = $v' package.json > _temp && mv _temp package.json; \ - fi - -# Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn workspaces focus --production && yarn cache clean && rm -rf ../**/src - -# Create final, arch specific, minimal size image. FROM node:18-alpine -COPY --from=builder /usr/src/yarn-project /usr/src/yarn-project -COPY --from=builder /usr/src/barretenberg/ts/package /usr/src/barretenberg/ts/package -COPY --from=builder /usr/src/noir/packages /usr/src/noir/packages -# Just until weird source-resolver bug fixed. -COPY --from=yarn-project /usr/src/noir/compiler /usr/src/noir/compiler +COPY --from=yarn-project-prod /usr/src /usr/src WORKDIR /usr/src/yarn-project/aztec-sandbox -ENV NODE_OPTIONS=--preserve-symlinks -ENTRYPOINT ["yarn"] -CMD [ "start" ] +ENTRYPOINT ["yarn", "start"] EXPOSE 8079 8080 diff --git a/yarn-project/aztec-sandbox/package.json b/yarn-project/aztec-sandbox/package.json index 0739a8d9cc4..99df402d9e8 100644 --- a/yarn-project/aztec-sandbox/package.json +++ b/yarn-project/aztec-sandbox/package.json @@ -15,7 +15,7 @@ }, "scripts": { "build": "yarn clean && tsc -b", - "start": "node --no-warnings ./dest/bin", + "start": "node --no-warnings --preserve-symlinks ./dest/bin", "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", diff --git a/yarn-project/cli/Dockerfile b/yarn-project/cli/Dockerfile index 2d8c842c90f..65ff0dee471 100644 --- a/yarn-project/cli/Dockerfile +++ b/yarn-project/cli/Dockerfile @@ -1,33 +1,18 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod # Need new arch specific image. -FROM node:18-alpine as builder -COPY --from=yarn-project /usr/src /usr/src -ARG COMMIT_TAG="" - -WORKDIR /usr/src/yarn-project/cli -RUN if [[ -n "${COMMIT_TAG}" ]]; then \ - jq --arg v ${COMMIT_TAG} '.version = $v' package.json > _temp && mv _temp package.json; \ - fi - -# Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn workspaces focus --production && yarn cache clean && rm -rf ../**/src - -# Create final, arch specific, minimal size image. FROM node:18-alpine -COPY --from=builder /usr/src/yarn-project /usr/src/yarn-project -COPY --from=builder /usr/src/barretenberg/ts/package /usr/src/barretenberg/ts/package -COPY --from=builder /usr/src/noir/packages /usr/src/noir/packages +COPY --from=yarn-project-prod /usr/src /usr/src +# Setup cache volume. ENV XDG_CACHE_HOME /cache RUN mkdir /cache && chmod 777 /cache VOLUME [ "/cache" ] +# Run as non-root user. RUN corepack enable - -# run as non-root user RUN addgroup -S aztec && adduser -S aztec -G aztec USER aztec -ENV NODE_OPTIONS="--no-warnings --preserve-symlinks" -ENTRYPOINT ["node", "/usr/src/yarn-project/cli/dest/bin/index.js"] \ No newline at end of file +WORKDIR /usr/src/yarn-project/cli +ENTRYPOINT ["yarn", "start"] \ No newline at end of file diff --git a/yarn-project/cli/package.json b/yarn-project/cli/package.json index 10e6102f95f..67f2fbd745b 100644 --- a/yarn-project/cli/package.json +++ b/yarn-project/cli/package.json @@ -20,7 +20,7 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests", - "start": "node --no-warnings ./dest/bin/index.js" + "start": "node --no-warnings --preserve-symlinks ./dest/bin/index.js" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/end-to-end/Dockerfile b/yarn-project/end-to-end/Dockerfile index 15f21a82149..11becb19792 100644 --- a/yarn-project/end-to-end/Dockerfile +++ b/yarn-project/end-to-end/Dockerfile @@ -6,7 +6,7 @@ RUN yarn build:web WORKDIR /usr/src/yarn-project/end-to-end # Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn cache clean && yarn workspaces focus --production +RUN yarn workspaces focus --production && yarn cache clean # Create final, minimal size image. # TODO: Not very minimal as chromium adds about 500MB of bloat :/ Separate or install at test runtime? diff --git a/yarn-project/scripts/version_packages.sh b/yarn-project/scripts/version_packages.sh new file mode 100755 index 00000000000..a708cb7a4ed --- /dev/null +++ b/yarn-project/scripts/version_packages.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -eu + +if [ -n "$COMMIT_TAG" ]; then + for workspace in $(yarn workspaces list --json | jq -r '.location'); do + (cd $workspace && jq --arg v $COMMIT_TAG '.version = $v' package.json > _temp && mv _temp package.json) + done +fi \ No newline at end of file From 62a82f1cee486be40705694e2dbb5653ae9f1542 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 30 Nov 2023 14:55:28 +0000 Subject: [PATCH 02/37] wip --- yarn-project/.dockerignore | 2 +- yarn-project/Dockerfile.prod | 3 ++- yarn-project/aztec-sandbox/package.json | 2 +- yarn-project/cli/Dockerfile | 9 ++++++--- yarn-project/cli/package.json | 2 +- yarn-project/cli/src/utils.ts | 8 ++++---- yarn-project/end-to-end/Dockerfile | 1 - .../src/contracts/test_contract/src/interface.nr | 9 +++++++-- yarn-project/yarn-project-base/Dockerfile | 5 ++++- 9 files changed, 26 insertions(+), 15 deletions(-) diff --git a/yarn-project/.dockerignore b/yarn-project/.dockerignore index 20f1f5070d4..c227696aa75 100644 --- a/yarn-project/.dockerignore +++ b/yarn-project/.dockerignore @@ -10,7 +10,7 @@ **/*.tsbuildinfo **/Dockerfile* **/node_modules -Dockerfile +Dockerfile* noir-contracts/src/types noir-contracts/src/artifacts diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index e9c9bbaef63..13832bc6583 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -15,4 +15,5 @@ COPY --from=yarn-project /usr/src/yarn-project /usr/src/yarn-project COPY --from=yarn-project /usr/src/barretenberg/ts/package /usr/src/barretenberg/ts/package COPY --from=yarn-project /usr/src/noir/packages /usr/src/noir/packages # Just until weird source-resolver bug fixed. -COPY --from=yarn-project /usr/src/noir/compiler /usr/src/noir/compiler \ No newline at end of file +COPY --from=yarn-project /usr/src/noir/compiler /usr/src/noir/compiler +RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules \ No newline at end of file diff --git a/yarn-project/aztec-sandbox/package.json b/yarn-project/aztec-sandbox/package.json index 99df402d9e8..0739a8d9cc4 100644 --- a/yarn-project/aztec-sandbox/package.json +++ b/yarn-project/aztec-sandbox/package.json @@ -15,7 +15,7 @@ }, "scripts": { "build": "yarn clean && tsc -b", - "start": "node --no-warnings --preserve-symlinks ./dest/bin", + "start": "node --no-warnings ./dest/bin", "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", diff --git a/yarn-project/cli/Dockerfile b/yarn-project/cli/Dockerfile index 65ff0dee471..fe13e02e9a9 100644 --- a/yarn-project/cli/Dockerfile +++ b/yarn-project/cli/Dockerfile @@ -2,6 +2,12 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-proj # Need new arch specific image. FROM node:18-alpine + +# Set Tini as the default entrypoint, to handle ctrl-c etc. +# Why not just yarn start? About 1 second difference in startup time. +# At time of writing it still takes 1.7s to just get the help to print. Needs investigating. +RUN apk add --no-cache tini +ENTRYPOINT ["/sbin/tini", "--", "node", "--no-warnings", "/usr/src/yarn-project/cli/dest/bin/index.js"] COPY --from=yarn-project-prod /usr/src /usr/src # Setup cache volume. @@ -13,6 +19,3 @@ VOLUME [ "/cache" ] RUN corepack enable RUN addgroup -S aztec && adduser -S aztec -G aztec USER aztec - -WORKDIR /usr/src/yarn-project/cli -ENTRYPOINT ["yarn", "start"] \ No newline at end of file diff --git a/yarn-project/cli/package.json b/yarn-project/cli/package.json index 67f2fbd745b..10e6102f95f 100644 --- a/yarn-project/cli/package.json +++ b/yarn-project/cli/package.json @@ -20,7 +20,7 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests", - "start": "node --no-warnings --preserve-symlinks ./dest/bin/index.js" + "start": "node --no-warnings ./dest/bin/index.js" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/cli/src/utils.ts b/yarn-project/cli/src/utils.ts index 9973d9fa2fc..dc4f8732617 100644 --- a/yarn-project/cli/src/utils.ts +++ b/yarn-project/cli/src/utils.ts @@ -17,7 +17,7 @@ import { import { LogId } from '@aztec/types'; import { CommanderError, InvalidArgumentError } from 'commander'; -import fs from 'fs'; +import { readFile, rename, writeFile } from 'fs/promises'; import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; import { encodeArgs } from './encoding.js'; @@ -107,7 +107,7 @@ export async function getContractArtifact(fileDir: string, log: LogFn) { } try { - contents = fs.readFileSync(fileDir, 'utf8'); + contents = await readFile(fileDir, 'utf8'); } catch { throw Error(`Contract ${fileDir} not found`); } @@ -412,11 +412,11 @@ export function parseFields(fields: string[]): Fr[] { export async function atomicUpdateFile(filePath: string, contents: string) { const tmpFilepath = filePath + '.tmp'; try { - await fs.promises.writeFile(tmpFilepath, contents, { + await writeFile(tmpFilepath, contents, { // let's crash if the tmp file already exists flag: 'wx', }); - await fs.promises.rename(tmpFilepath, filePath); + await rename(tmpFilepath, filePath); } catch (e) { if (e instanceof Error && 'code' in e && e.code === 'EEXIST') { const commanderError = new CommanderError( diff --git a/yarn-project/end-to-end/Dockerfile b/yarn-project/end-to-end/Dockerfile index 11becb19792..8d6496615f9 100644 --- a/yarn-project/end-to-end/Dockerfile +++ b/yarn-project/end-to-end/Dockerfile @@ -29,5 +29,4 @@ COPY --from=builder /usr/src/yarn-project/aztec.js/dest/main.js /usr/src/yarn-pr WORKDIR /usr/src/yarn-project/end-to-end -ENV NODE_OPTIONS=--preserve-symlinks ENTRYPOINT ["yarn", "test"] diff --git a/yarn-project/noir-contracts/src/contracts/test_contract/src/interface.nr b/yarn-project/noir-contracts/src/contracts/test_contract/src/interface.nr index 65cf52e96b5..1bb62e9f3f6 100644 --- a/yarn-project/noir-contracts/src/contracts/test_contract/src/interface.nr +++ b/yarn-project/noir-contracts/src/contracts/test_contract/src/interface.nr @@ -1,5 +1,5 @@ /* Autogenerated file, do not edit! */ - + use dep::std; use dep::aztec::context::{ PrivateContext, PublicContext }; use dep::aztec::constants_gen::RETURN_VALUES_LENGTH; @@ -26,6 +26,7 @@ struct ManyNotesADeepStructTestCodeGenStruct { secret_hash: Field, } + // Interface for calling Test functions from a private context struct TestPrivateContextInterface { address: Field, @@ -241,6 +242,9 @@ impl TestPrivateContextInterface { } } + + + // Interface for calling Test functions from a public context struct TestPublicContextInterface { @@ -326,4 +330,5 @@ impl TestPublicContextInterface { } } - + + diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index a4ee8c6b91e..1e22c3c76db 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -66,7 +66,10 @@ RUN mkdir /usr/src/noir/compiler && mv /usr/src/noir/packages/source-resolver /u # We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they # walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution # correctly for portalled packages, is to use --preserve-symlinks when running node. -# This does work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic +# by initialising the module more than once. +# It's worth noting this is only needed in these alpine containers. RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules WORKDIR /usr/src/yarn-project From cbc30f657464bde5bf5ab4d1f24a77e7747e8540 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 30 Nov 2023 16:32:50 +0000 Subject: [PATCH 03/37] wip --- yarn-project/Dockerfile.prod | 10 ++++------ yarn-project/yarn-project-base/Dockerfile | 3 +-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index 13832bc6583..631da312d10 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -9,11 +9,9 @@ RUN yarn workspaces focus @aztec/cli @aztec/aztec-sandbox @aztec/aztec-faucet -- yarn cache clean && \ rm -rf ./**/src +# We no longer need nargo. +RUN rm -rf /usr/src/noir/target + # Create fresh minimal size image. FROM node:18-alpine -COPY --from=yarn-project /usr/src/yarn-project /usr/src/yarn-project -COPY --from=yarn-project /usr/src/barretenberg/ts/package /usr/src/barretenberg/ts/package -COPY --from=yarn-project /usr/src/noir/packages /usr/src/noir/packages -# Just until weird source-resolver bug fixed. -COPY --from=yarn-project /usr/src/noir/compiler /usr/src/noir/compiler -RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules \ No newline at end of file +COPY --from=yarn-project /usr/src /usr/src \ No newline at end of file diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index 1e22c3c76db..a6ed174d487 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -68,8 +68,7 @@ RUN mkdir /usr/src/noir/compiler && mv /usr/src/noir/packages/source-resolver /u # correctly for portalled packages, is to use --preserve-symlinks when running node. # This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. # Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic -# by initialising the module more than once. -# It's worth noting this is only needed in these alpine containers. +# by initialising the module more than once. So at present I don't see a viable alternative. RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules WORKDIR /usr/src/yarn-project From e4cb78a95f90cdc617b23188a3a472189028f270 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 30 Nov 2023 16:59:39 +0000 Subject: [PATCH 04/37] Rebuild patterns and fix. --- .circleci/config.yml | 1 + build_manifest.yml | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b9ef6007d4..2319eafca10 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1085,6 +1085,7 @@ workflows: requires: - yarn-project-base <<: *defaults + - yarn-project-prod: *defaults_yarn_project - yarn-project-formatting: *defaults_yarn_project - yarn-project-tests: *defaults_yarn_project - end-to-end: *defaults_yarn_project diff --git a/build_manifest.yml b/build_manifest.yml index 337992d1786..09f4f487367 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -1,10 +1,38 @@ noir: - # Add less aggressive rebuildPatterns. buildDir: noir + rebuildPatterns: + - ^noir/acvm-repo + - ^noir/compiler + - ^noir/aztec_macros + - ^noir/noir_stdlib + - ^noir/tooling/backend_interface + - ^noir/tooling/bb_abstraction_leaks + - ^noir/tooling/debugger + - ^noir/tooling/lsp + - ^noir/tooling/nargo + - ^noir/tooling/nargo_cli + - ^noir/tooling/nargo_toml + - ^noir/tooling/nargo_fmt + - ^noir/tooling/noirc_abi noir-packages: buildDir: noir dockerfile: Dockerfile.packages + rebuildPatterns: + - ^noir/.yarn + - ^noir/.yarnrc.yml + - ^noir/package.json + - ^noir/yarn.lock + - ^noir/acvm-repo + - ^noir/compiler + - ^noir/aztec_macros + - ^noir/noir_stdlib + - ^noir/tooling/noir_codegen + - ^noir/tooling/noir_js + - ^noir/tooling/noir_js_backend_barretenberg + - ^noir/tooling/noir_js_types + - ^noir/tooling/noirc_abi + - ^noir/tooling/noirc_abi_wasm noir-acir-tests: buildDir: noir/test_programs From dd2af063f152b815fd70a27628795c7ef72a5d76 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 30 Nov 2023 18:05:07 +0000 Subject: [PATCH 05/37] Force deploy comtainer [ci force-deploy] --- build-system/scripts/deploy_dockerhub | 35 +++++++++++++++------------ build-system/scripts/should_deploy | 10 +++++--- build_manifest.yml | 2 ++ yarn-project/deploy_npm.sh | 5 ++++ 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 4009f453d48..047980966f5 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -4,10 +4,7 @@ set -eu REPOSITORY=$1 # TODO: Why even provide this? We can just figure it out by probing. -ARCH_LIST=${2:-} - -# The tag to use for release images. Can be turned into an argument if needed, for now we only release 'latest'. -DIST_TAG="latest" +ARCH_LIST=$2 function docker_or_dryrun { if [ "$DRY_DEPLOY" -eq 1 ]; then @@ -20,10 +17,13 @@ function docker_or_dryrun { echo "Repo: $REPOSITORY" echo "Arch List: $ARCH_LIST" -VERSION_TAG=$(extract_tag_version $REPOSITORY true) +if [ -n "$COMMIT_TAG" ]; then + TAG=$(extract_tag_version $REPOSITORY true) +else + TAG=$(echo "$BRANCH" | sed 's/[^a-zA-Z0-9_.-]/_/g') +fi -MANIFEST_DEPLOY_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$VERSION_TAG -MANIFEST_DIST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$DIST_TAG +MANIFEST_DEPLOY_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$TAG # Login to dockerhub and ecr dockerhub_login @@ -35,19 +35,22 @@ for ARCH in $ARCH_LIST; do retry docker pull $IMAGE_COMMIT_URI # Retag and push image. - IMAGE_DEPLOY_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$VERSION_TAG-$ARCH + IMAGE_DEPLOY_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$TAG-$ARCH docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_URI docker_or_dryrun push $IMAGE_DEPLOY_URI - echo "Adding image $IMAGE_DEPLOY_URI to manifest list $MANIFEST_DEPLOY_URI..." + echo "Adding image $IMAGE_DEPLOY_URI to manifest $MANIFEST_DEPLOY_URI..." docker_or_dryrun manifest create $MANIFEST_DEPLOY_URI --amend $IMAGE_DEPLOY_URI - - echo "Adding image $IMAGE_DEPLOY_URI to manifest list $MANIFEST_DIST_URI" - docker_or_dryrun manifest create $MANIFEST_DIST_URI --amend $IMAGE_DEPLOY_URI done -echo "Tagging $MANIFEST_DEPLOY_URI as $VERSION_TAG..." +echo "Pushing manifest $MANIFEST_DEPLOY_URI..." docker_or_dryrun manifest push --purge $MANIFEST_DEPLOY_URI -# Publish version as latest. -echo "Tagging $MANIFEST_DEPLOY_URI as $DIST_TAG..." -docker_or_dryrun manifest push --purge $MANIFEST_DIST_URI + +if [ -n "$COMMIT_TAG" ]; then + # Publish as latest. + LATEST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:latest + echo "Tagging $MANIFEST_DEPLOY_URI as $LATEST_URI..." + docker_or_dryrun pull $MANIFEST_DEPLOY_URI + docker_or_dryrun tag $MANIFEST_DEPLOY_URI $LATEST_URI + docker_or_dryrun push $LATEST_URI +fi diff --git a/build-system/scripts/should_deploy b/build-system/scripts/should_deploy index aac1504ab33..7c7143de3d0 100755 --- a/build-system/scripts/should_deploy +++ b/build-system/scripts/should_deploy @@ -1,7 +1,9 @@ #!/bin/bash -# Retuns success if we are expected to do a deploy. -# At present this is only if we have a commit tag. -# Once we are doing master deployments to devnet, we'll want to check if BRANCH is master. +# Returns true if we are expected to proceed with a deploy job. +# Specifically if we have a commit tag, are master, or are being forced to deploy. +# This script should be used at the start of all deployment steps to early out PR runs. +# Later deployment steps may early out regardless, e.g. npm deploys only run for tagged commits. +# Dockerhub publishing will publish for master, or PR's run with force-deploy, but only tag latest on tagged commits. set -eu -[ -n "$COMMIT_TAG" ] \ No newline at end of file +[[ -n "$COMMIT_TAG" || "$BRANCH" == "master" || "$COMMIT_MESSAGE" == *"[ci force-deploy]"* ]] \ No newline at end of file diff --git a/build_manifest.yml b/build_manifest.yml index 09f4f487367..a49fc276196 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -1,6 +1,7 @@ noir: buildDir: noir rebuildPatterns: + - ^noir/Dockerfile - ^noir/acvm-repo - ^noir/compiler - ^noir/aztec_macros @@ -19,6 +20,7 @@ noir-packages: buildDir: noir dockerfile: Dockerfile.packages rebuildPatterns: + - ^noir/Dockerfile.packages - ^noir/.yarn - ^noir/.yarnrc.yml - ^noir/package.json diff --git a/yarn-project/deploy_npm.sh b/yarn-project/deploy_npm.sh index 20575d12167..029eb3a135b 100755 --- a/yarn-project/deploy_npm.sh +++ b/yarn-project/deploy_npm.sh @@ -2,6 +2,11 @@ [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu +if [ -z "$COMMIT_TAG" ]; then + echo "No commit tag, not deploying to npm." + exit 0 +fi + extract_repo yarn-project /usr/src project cd project/src/yarn-project From 10172b7662e32d065a9d65e38da6e99dcc13d20f Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 30 Nov 2023 18:46:33 +0000 Subject: [PATCH 06/37] Leading layers must be same between output containers. [ci force-deploy] --- yarn-project/aztec-sandbox/Dockerfile | 1 + yarn-project/cli/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/yarn-project/aztec-sandbox/Dockerfile b/yarn-project/aztec-sandbox/Dockerfile index 810a1e23e83..6b47e6cc638 100644 --- a/yarn-project/aztec-sandbox/Dockerfile +++ b/yarn-project/aztec-sandbox/Dockerfile @@ -3,6 +3,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-proj # Need new arch specific image. FROM node:18-alpine COPY --from=yarn-project-prod /usr/src /usr/src + WORKDIR /usr/src/yarn-project/aztec-sandbox ENTRYPOINT ["yarn", "start"] EXPOSE 8079 8080 diff --git a/yarn-project/cli/Dockerfile b/yarn-project/cli/Dockerfile index fe13e02e9a9..09fff65eea4 100644 --- a/yarn-project/cli/Dockerfile +++ b/yarn-project/cli/Dockerfile @@ -2,13 +2,13 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-proj # Need new arch specific image. FROM node:18-alpine +COPY --from=yarn-project-prod /usr/src /usr/src # Set Tini as the default entrypoint, to handle ctrl-c etc. # Why not just yarn start? About 1 second difference in startup time. # At time of writing it still takes 1.7s to just get the help to print. Needs investigating. RUN apk add --no-cache tini ENTRYPOINT ["/sbin/tini", "--", "node", "--no-warnings", "/usr/src/yarn-project/cli/dest/bin/index.js"] -COPY --from=yarn-project-prod /usr/src /usr/src # Setup cache volume. ENV XDG_CACHE_HOME /cache From 1187702653fb8dc3c2b4ef805c7006e890c24e4b Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 30 Nov 2023 22:13:52 +0000 Subject: [PATCH 07/37] Lets play with buildx [ci rebuild yarn-project-prod] --- .circleci/config.yml | 122 ++++++++++++----------- build-system/scripts/build | 16 ++- build-system/scripts/calculate_image_uri | 13 ++- build-system/scripts/query_manifest | 3 + build_manifest.yml | 2 + 5 files changed, 91 insertions(+), 65 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2319eafca10..541f4979d2c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -379,7 +379,7 @@ jobs: name: Test command: cond_spot_run_container yarn-project 64 test | add_timestamps - aztec-sandbox-x86_64: + aztec-sandbox: machine: image: ubuntu-2204:2023.07.2 resource_class: large @@ -390,22 +390,22 @@ jobs: name: "Build and test" command: build aztec-sandbox - aztec-sandbox-arm64: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: arm.large - steps: - - *checkout - - *setup_env - - run: - name: "Build and test" - # We need to force not to use docker buildkit because for some reason on arm only, it ends up making a call - # out to eu-west2 despite the image being locally tagged, resulting in unauthorized 401. Weird docker bug? - command: | - echo "export DOCKER_BUILDKIT=" > $BASH_ENV - build aztec-sandbox - - cli-x86_64: + # aztec-sandbox-arm64: + # machine: + # image: ubuntu-2204:2023.07.2 + # resource_class: arm.large + # steps: + # - *checkout + # - *setup_env + # - run: + # name: "Build and test" + # # We need to force not to use docker buildkit because for some reason on arm only, it ends up making a call + # # out to eu-west2 despite the image being locally tagged, resulting in unauthorized 401. Weird docker bug? + # command: | + # echo "export DOCKER_BUILDKIT=" > $BASH_ENV + # build aztec-sandbox + + cli: machine: image: ubuntu-2204:2023.07.2 resource_class: large @@ -416,20 +416,20 @@ jobs: name: "Build and test" command: build cli - cli-arm64: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: arm.large - steps: - - *checkout - - *setup_env - - run: - name: "Build and test" - # We need to force not to use docker buildkit because for some reason on arm only, it ends up making a call - # out to eu-west2 despite the image being locally tagged, resulting in unauthorized 401. Weird docker bug? - command: | - echo "export DOCKER_BUILDKIT=" > $BASH_ENV - build cli + # cli-arm64: + # machine: + # image: ubuntu-2204:2023.07.2 + # resource_class: arm.large + # steps: + # - *checkout + # - *setup_env + # - run: + # name: "Build and test" + # # We need to force not to use docker buildkit because for some reason on arm only, it ends up making a call + # # out to eu-west2 despite the image being locally tagged, resulting in unauthorized 401. Weird docker bug? + # command: | + # echo "export DOCKER_BUILDKIT=" > $BASH_ENV + # build cli mainnet-fork: machine: @@ -453,18 +453,18 @@ jobs: name: "Build and test" command: build aztec-faucet | add_timestamps - ecr-manifest: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large - steps: - - *checkout - - *setup_env - - run: - name: "Create ECR manifest" - command: | - create_ecr_manifest aztec-sandbox x86_64,arm64 - create_ecr_manifest cli x86_64,arm64 + # ecr-manifest: + # machine: + # image: ubuntu-2204:2023.07.2 + # resource_class: large + # steps: + # - *checkout + # - *setup_env + # - run: + # name: "Create ECR manifest" + # command: | + # create_ecr_manifest aztec-sandbox x86_64,arm64 + # create_ecr_manifest cli x86_64,arm64 boxes-blank-react: machine: @@ -1090,38 +1090,44 @@ workflows: - yarn-project-tests: *defaults_yarn_project - end-to-end: *defaults_yarn_project - build-docs: *defaults_yarn_project - - aztec-sandbox-x86_64: *defaults_yarn_project_prod - - aztec-sandbox-arm64: *defaults_yarn_project_prod - - cli-x86_64: *defaults_yarn_project_prod - - cli-arm64: *defaults_yarn_project_prod + + # Artifacts + - aztec-sandbox: *defaults_yarn_project_prod + - cli: *defaults_yarn_project_prod - aztec-faucet: *defaults_yarn_project_prod - - ecr-manifest: - requires: - - aztec-sandbox-x86_64 - - aztec-sandbox-arm64 - - cli-x86_64 - - cli-arm64 - <<: *defaults + # - aztec-sandbox-x86_64: *defaults_yarn_project_prod + # - aztec-sandbox-arm64: *defaults_yarn_project_prod + # - cli-x86_64: *defaults_yarn_project_prod + # - cli-arm64: *defaults_yarn_project_prod + # - aztec-faucet: *defaults_yarn_project_prod + # - ecr-manifest: + # requires: + # - aztec-sandbox-x86_64 + # - aztec-sandbox-arm64 + # - cli-x86_64 + # - cli-arm64 + # <<: *defaults # Boxes. - boxes-blank-react: requires: - - aztec-sandbox-x86_64 + - aztec-sandbox <<: *defaults - boxes-blank: requires: - - aztec-sandbox-x86_64 + - aztec-sandbox <<: *defaults - boxes-token: requires: - - aztec-sandbox-x86_64 + - aztec-sandbox <<: *defaults # End to end tests. - e2e-join: requires: - end-to-end - - ecr-manifest + - aztec-sandbox + - cli <<: *defaults - e2e-2-pxes: *e2e_test - e2e-deploy-contract: *e2e_test diff --git a/build-system/scripts/build b/build-system/scripts/build index 71826e74dbf..8cebeea29ec 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -108,8 +108,14 @@ COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) echo "Commit tag version: $COMMIT_TAG_VERSION" # Build the actual image and give it a commit tag. -IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) -echo "Building image: $IMAGE_COMMIT_URI" -docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . -echo "Pushing image: $IMAGE_COMMIT_URI" -retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 +if [ "$(query_manifest multiarch $REPOSITORY)" == "true" ]; then + IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY "") + echo "Building image: $IMAGE_COMMIT_URI" + docker buildx build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH --platform linux/amd64,linux/arm64 . --push +else + IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) + echo "Building image: $IMAGE_COMMIT_URI" + docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . + echo "Pushing image: $IMAGE_COMMIT_URI" + retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 +fi diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index fe60a9c05ab..ed0eb24a944 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -3,7 +3,16 @@ set -eu REPOSITORY=$1 -ARCH=${2:-$(uname -m)} +# Only probe if $2 is unset. +ARCH=${2-$(uname -m)} + +# Normalise. [ "$ARCH" == "aarch64" ] && ARCH=arm64 + CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$ARCH" + +if [ -z "$ARCH" ]; then + echo "$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH" +else + echo "$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$ARCH" +fi diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index 0358fbf89cb..cd0b5c0888b 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -126,4 +126,7 @@ case "$CMD" in if [ "$(git ls-tree HEAD $DIR | awk '{print $2}')" = "commit" ]; then git ls-tree HEAD $DIR | awk '{print $4}' fi + ;; + multiarch) + yq -r ".\"$REPO\".multiarch // false" $MANIFEST esac diff --git a/build_manifest.yml b/build_manifest.yml index a49fc276196..ade4de7c9aa 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -139,6 +139,7 @@ aztec-sandbox: projectDir: yarn-project/aztec-sandbox dependencies: - yarn-project-prod + multiarch: true aztec-faucet: buildDir: yarn-project @@ -151,6 +152,7 @@ cli: projectDir: yarn-project/cli dependencies: - yarn-project-prod + multiarch: true boxes-blank-react: buildDir: yarn-project From 7a3206dc650c4efd668857f8e580c1f4beaae506 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 30 Nov 2023 22:17:58 +0000 Subject: [PATCH 08/37] Lets play with buildx [ci rebuild sandbox] [ci rebuild cli] From 5667550d18df077f205f9544e72c3415386cfa4e Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 30 Nov 2023 22:21:38 +0000 Subject: [PATCH 09/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] --- build-system/scripts/build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-system/scripts/build b/build-system/scripts/build index 8cebeea29ec..217737e6f59 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -111,6 +111,8 @@ echo "Commit tag version: $COMMIT_TAG_VERSION" if [ "$(query_manifest multiarch $REPOSITORY)" == "true" ]; then IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY "") echo "Building image: $IMAGE_COMMIT_URI" + docker buildx create --name builder --use + docker buildx inspect --bootstrap docker buildx build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH --platform linux/amd64,linux/arm64 . --push else IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) From 2901ba293b066697e9bd607d1ef4c2753128a868 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 11:49:39 +0000 Subject: [PATCH 10/37] wip --- build-system/scripts/build | 38 +++++++++++++++--------- build-system/scripts/calculate_image_tag | 13 ++++++-- build-system/scripts/calculate_image_uri | 16 ++-------- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index 217737e6f59..3b91092a4db 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -86,22 +86,32 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then popd fi -# For each dependency, pull in the latest image and give it correct tag. +# For each dependency, substitute references to the dependency in dockerfile, with the relevent built image uri. +# We have to perform a bit of probing to determine which actual image we want to use. +# When we used buildx to create a multiarch image, there will be no images with "-$ARCH" suffixes (normalise this?). +# Also we sometimes build an arm image from an x86 parent, so there won't always be an arm parent, and we fallback. for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do - PARENT_IMAGE_URI=$(calculate_image_uri $PARENT_REPO) - echo "Pulling dependency $PARENT_IMAGE_URI..." - if ! fetch_image $PARENT_IMAGE_URI; then - # This is a *bit* of a hack maybe. Some of our arm images can be built from x86 dependents. - # e.g. node projects are architecture independent. - # This may not hold true if we start introducing npm modules that are backed by native code. - # But for now, to avoid building some projects twice, we can fallback onto x86 variant. - PARENT_IMAGE_URI=$(calculate_image_uri $PARENT_REPO x86_64) - echo "Falling back onto x86 build. Pulling dependency $PARENT_IMAGE_URI..." - fetch_image $PARENT_IMAGE_URI + # We want the parent image tag without any arch suffix. + PARENT_IMAGE_TAG=$(calculate_image_tag $PARENT_REPO "") + + # Attempt to locate multiarch image. + if ! image_exists $PARENT_REPO $PARENT_IMAGE_TAG; then + # Attempt to locate our specific arch image. + PARENT_IMAGE_TAG=$(calculate_image_tag $PARENT_REPO) + if ! image_exists $PARENT_REPO $PARENT_IMAGE_TAG; then + # Finally attempt to locate x86_64 image tag, as sometimes we build arch specific images from x86_64 images. + PARENT_IMAGE_TAG=$(calculate_image_tag $PARENT_REPO x86_64) + if ! image_exists $PARENT_REPO $PARENT_IMAGE_TAG; then + echo "Failed to locate multiarch image, arch specific image, or x86_64 image. Aborting." + exit 1 + fi + fi fi - # Tag it to look like an official release as that's what we use in Dockerfiles. - TAG=$ECR_DEPLOY_URL/$PARENT_REPO - docker tag $PARENT_IMAGE_URI $TAG + + # Substitute references to parent repo, with the relevent built image uri. + DEPLOY_URI=$ECR_DEPLOY_URL/$PARENT_REPO + PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:$PARENT_IMAGE_TAG + sed -i "s|^FROM $DEPLOY_URI|FROM $PARENT_IMAGE_URI|" $DOCKERFILE done COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) diff --git a/build-system/scripts/calculate_image_tag b/build-system/scripts/calculate_image_tag index 2a7786ecdc1..0fc2e3be4d5 100755 --- a/build-system/scripts/calculate_image_tag +++ b/build-system/scripts/calculate_image_tag @@ -3,7 +3,16 @@ set -eu REPOSITORY=$1 -ARCH=${2:-$(uname -m)} +# Only probe if $2 is unset. +ARCH=${2-$(uname -m)} + +# Normalise. [ "$ARCH" == "aarch64" ] && ARCH=arm64 + CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "cache-$CONTENT_HASH-$ARCH" + +if [ -z "$ARCH" ]; then + echo "cache-$CONTENT_HASH" +else + echo "cache-$CONTENT_HASH-$ARCH" +fi diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index ed0eb24a944..7e606a227e7 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -2,17 +2,5 @@ [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu -REPOSITORY=$1 -# Only probe if $2 is unset. -ARCH=${2-$(uname -m)} - -# Normalise. -[ "$ARCH" == "aarch64" ] && ARCH=arm64 - -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) - -if [ -z "$ARCH" ]; then - echo "$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH" -else - echo "$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$ARCH" -fi +TAG=$(calculate_image_tag $@) +echo "$ECR_URL/$REPOSITORY:$TAG" From 307111b895d107056f7db487b85b47764abd8310 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 12:13:06 +0000 Subject: [PATCH 11/37] wip --- build-system/scripts/calculate_image_uri | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index 7e606a227e7..f12fb033b48 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -2,5 +2,7 @@ [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu +REPOSITORY=$1 + TAG=$(calculate_image_tag $@) echo "$ECR_URL/$REPOSITORY:$TAG" From d7cd82f21a5e35042c023c407013ef842c33c4ba Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 12:15:53 +0000 Subject: [PATCH 12/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] From b2169890f466c490471fa35932b848dad1a68451 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 12:26:54 +0000 Subject: [PATCH 13/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] --- build-system/scripts/calculate_image_uri | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index f12fb033b48..8efd7ab4cdc 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -4,5 +4,5 @@ set -eu REPOSITORY=$1 -TAG=$(calculate_image_tag $@) +TAG=$(calculate_image_tag "$@") echo "$ECR_URL/$REPOSITORY:$TAG" From 845d6a7cf20d7f3368d65e408e0c35967ce72630 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 13:13:54 +0000 Subject: [PATCH 14/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] --- build_manifest.yml | 1 + yarn-project/Dockerfile.prod | 3 +++ yarn-project/aztec-sandbox/Dockerfile | 4 ---- yarn-project/cli/Dockerfile | 4 ---- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/build_manifest.yml b/build_manifest.yml index ade4de7c9aa..903181a47f2 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -133,6 +133,7 @@ yarn-project-prod: - ^yarn-project/Dockerfile.prod dependencies: - yarn-project + multiarch: true aztec-sandbox: buildDir: yarn-project diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index 631da312d10..e529b366409 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -2,6 +2,9 @@ # we then generate downstream multiarch containers to execute the specific projects. FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project +# Need new arch specific image. +FROM node:18-alpine +COPY --from=yarn-project-prod /usr/src /usr/src ARG COMMIT_TAG="" RUN ./scripts/version_packages.sh # Productionify. See comment in yarn-project-base/Dockerfile. diff --git a/yarn-project/aztec-sandbox/Dockerfile b/yarn-project/aztec-sandbox/Dockerfile index 6b47e6cc638..40731ebdcab 100644 --- a/yarn-project/aztec-sandbox/Dockerfile +++ b/yarn-project/aztec-sandbox/Dockerfile @@ -1,9 +1,5 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod -# Need new arch specific image. -FROM node:18-alpine -COPY --from=yarn-project-prod /usr/src /usr/src - WORKDIR /usr/src/yarn-project/aztec-sandbox ENTRYPOINT ["yarn", "start"] EXPOSE 8079 8080 diff --git a/yarn-project/cli/Dockerfile b/yarn-project/cli/Dockerfile index 09fff65eea4..0b8570793f7 100644 --- a/yarn-project/cli/Dockerfile +++ b/yarn-project/cli/Dockerfile @@ -1,9 +1,5 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod -# Need new arch specific image. -FROM node:18-alpine -COPY --from=yarn-project-prod /usr/src /usr/src - # Set Tini as the default entrypoint, to handle ctrl-c etc. # Why not just yarn start? About 1 second difference in startup time. # At time of writing it still takes 1.7s to just get the help to print. Needs investigating. From c6c17468776c8f3bc0b5d58567a1eb31233e47e1 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 13:26:18 +0000 Subject: [PATCH 15/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] --- yarn-project/Dockerfile.prod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index e529b366409..a0b227b263d 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -3,7 +3,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project # Need new arch specific image. -FROM node:18-alpine +FROM node:18-alpine AS builder COPY --from=yarn-project-prod /usr/src /usr/src ARG COMMIT_TAG="" RUN ./scripts/version_packages.sh @@ -17,4 +17,4 @@ RUN rm -rf /usr/src/noir/target # Create fresh minimal size image. FROM node:18-alpine -COPY --from=yarn-project /usr/src /usr/src \ No newline at end of file +COPY --from=builder /usr/src /usr/src \ No newline at end of file From 4dbe5d8975938ee80007b8cb0f827b7249727dac Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 13:39:58 +0000 Subject: [PATCH 16/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] --- build-system/scripts/calculate_content_hash | 21 ++++---------------- build-system/scripts/calculate_rebuild_files | 20 +++++++++++++++++++ build_manifest.yml | 2 +- yarn-project/Dockerfile.prod | 4 ++-- 4 files changed, 27 insertions(+), 20 deletions(-) create mode 100755 build-system/scripts/calculate_rebuild_files diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 630595aa371..28ae8ff2e62 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -1,21 +1,8 @@ #!/bin/bash - [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -REPOSITORY=$1 -COMMIT_HASH=${2:-${COMMIT_HASH:-$(git rev-parse HEAD)}} - -# Get list of rebuild patterns, concat them with regex 'or' (|), and double escape \ for awk -v. -AWK_PATTERN=$(query_manifest rebuildPatterns $REPOSITORY | tr '\n' '|' | sed 's/\\/\\\\/g') -# Remove the trailing '|'. -AWK_PATTERN=${AWK_PATTERN%|} - -cd "$(git rev-parse --show-toplevel)" +set -euo pipefail -# an example line is +# An example line is: # 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json -# this format is beneficial as it grabs the hashes from git efficiently -# we will next filter by our rebuild patterns -# then we pipe the hash portion of each file to git hash-object to produce our content hash -git ls-tree -r $COMMIT_HASH | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin \ No newline at end of file +# Extract the hashes and pipe the hash portion of each file to git hash-object to produce our content hash. +calculate_rebuild_files "$@" | awk '{print $3}' | git hash-object --stdin \ No newline at end of file diff --git a/build-system/scripts/calculate_rebuild_files b/build-system/scripts/calculate_rebuild_files new file mode 100755 index 00000000000..9be1afd4912 --- /dev/null +++ b/build-system/scripts/calculate_rebuild_files @@ -0,0 +1,20 @@ +#!/bin/bash + +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +COMMIT_HASH=${2:-${COMMIT_HASH:-$(git rev-parse HEAD)}} + +# Get list of rebuild patterns, concat them with regex 'or' (|), and double escape \ for awk -v. +AWK_PATTERN=$(query_manifest rebuildPatterns $REPOSITORY | tr '\n' '|' | sed 's/\\/\\\\/g') +# Remove the trailing '|'. +AWK_PATTERN=${AWK_PATTERN%|} + +cd "$(git rev-parse --show-toplevel)" + +# An example line is: +# 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json +# This format is beneficial as it grabs the hashes from git efficiently. +# We then filter by our rebuild patterns. +git ls-tree -r $COMMIT_HASH | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $0}' \ No newline at end of file diff --git a/build_manifest.yml b/build_manifest.yml index 903181a47f2..48070b85796 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -122,7 +122,7 @@ yarn-project: buildDir: yarn-project rebuildPatterns: - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml)$ - - ^yarn-project/Dockerfile + - ^yarn-project/Dockerfile$ dependencies: - yarn-project-base diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index a0b227b263d..6aa88143616 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -3,8 +3,8 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project # Need new arch specific image. -FROM node:18-alpine AS builder -COPY --from=yarn-project-prod /usr/src /usr/src +FROM node:18-alpine +COPY --from=yarn-project /usr/src /usr/src ARG COMMIT_TAG="" RUN ./scripts/version_packages.sh # Productionify. See comment in yarn-project-base/Dockerfile. From 2b37d4f18e8a595dc7ca02554fdd91774466f946 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 13:54:53 +0000 Subject: [PATCH 17/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] --- yarn-project/Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index 6aa88143616..621aa71ece7 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -3,7 +3,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project # Need new arch specific image. -FROM node:18-alpine +FROM node:18-alpine AS builder COPY --from=yarn-project /usr/src /usr/src ARG COMMIT_TAG="" RUN ./scripts/version_packages.sh From c8b54e90d93896a403c4d1da9b0b655538df8a90 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 14:02:32 +0000 Subject: [PATCH 18/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] --- yarn-project/Dockerfile.prod | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index 621aa71ece7..474e046e3fe 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -5,6 +5,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project # Need new arch specific image. FROM node:18-alpine AS builder COPY --from=yarn-project /usr/src /usr/src +WORKDIR /usr/src/yarn-project ARG COMMIT_TAG="" RUN ./scripts/version_packages.sh # Productionify. See comment in yarn-project-base/Dockerfile. From 6f3787951539534330802402706eeb0242bf0453 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 14:15:31 +0000 Subject: [PATCH 19/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] --- yarn-project/Dockerfile.prod | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index 474e046e3fe..82c21c1755c 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -4,6 +4,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project # Need new arch specific image. FROM node:18-alpine AS builder +RUN apk add bash jq --no-cache COPY --from=yarn-project /usr/src /usr/src WORKDIR /usr/src/yarn-project ARG COMMIT_TAG="" From e507c115f124f33bfb499642b903aafbc6f49e6f Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 15:35:54 +0000 Subject: [PATCH 20/37] Lets play with buildx [ci rebuild noir] [ci rebuild aztec-sandbox] [ci rebuild cli] --- build-system/scripts/build | 22 ++++++++++++++++------ build-system/scripts/calculate_image_tag | 15 ++++++++++----- build_manifest.yml | 7 ++++--- cspell.json | 2 ++ 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index 3b91092a4db..9a64d153e5e 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -117,16 +117,26 @@ done COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) echo "Commit tag version: $COMMIT_TAG_VERSION" -# Build the actual image and give it a commit tag. -if [ "$(query_manifest multiarch $REPOSITORY)" == "true" ]; then - IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY "") - echo "Building image: $IMAGE_COMMIT_URI" +IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) +echo "Building image: $IMAGE_COMMIT_URI" + +MULTIARCH=$(query_manifest multiarch $REPOSITORY) + +# Build the image. +if [ "$MULTIARCH" == "buildx" ]; then + # We've requested to use buildx. This will build both arch containers on the host machine using virtualization. + # The result is a single image tag that supports multiarch. docker buildx create --name builder --use docker buildx inspect --bootstrap docker buildx build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH --platform linux/amd64,linux/arm64 . --push else - IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) - echo "Building image: $IMAGE_COMMIT_URI" + # If multiarch is set to "host", the assumption is that we doing multiple builds on different architcture machines + # in parallel, and that there is a another job that runs afterwards to combine them into a manifest. + # In this case we need to augment the image tag with this host architecture to ensure it's uniqueness. + if [ "$MULTIARCH" == "host" ]; then + IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY host) + fi + docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . echo "Pushing image: $IMAGE_COMMIT_URI" retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 diff --git a/build-system/scripts/calculate_image_tag b/build-system/scripts/calculate_image_tag index 0fc2e3be4d5..c273648287c 100755 --- a/build-system/scripts/calculate_image_tag +++ b/build-system/scripts/calculate_image_tag @@ -1,18 +1,23 @@ #!/bin/bash +# Return a repositories build cache image tag based on content hash. +# If the second argument is set: +# It's used to suffix the tag with the given unique arch descriptor. +# Unless it's "host" in which case use the host machines arch as the arch descriptor. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu REPOSITORY=$1 -# Only probe if $2 is unset. -ARCH=${2-$(uname -m)} - -# Normalise. -[ "$ARCH" == "aarch64" ] && ARCH=arm64 +ARCH=${2:-} CONTENT_HASH=$(calculate_content_hash $REPOSITORY) if [ -z "$ARCH" ]; then echo "cache-$CONTENT_HASH" else + if [ "$ARCH" == "host" ]; then + ARCH=$(uname -m) + fi + # Normalise. + [ "$ARCH" == "aarch64" ] && ARCH=arm64 echo "cache-$CONTENT_HASH-$ARCH" fi diff --git a/build_manifest.yml b/build_manifest.yml index 48070b85796..11b851106a6 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -15,6 +15,7 @@ noir: - ^noir/tooling/nargo_toml - ^noir/tooling/nargo_fmt - ^noir/tooling/noirc_abi + multiarch: host noir-packages: buildDir: noir @@ -133,14 +134,14 @@ yarn-project-prod: - ^yarn-project/Dockerfile.prod dependencies: - yarn-project - multiarch: true + multiarch: buildx aztec-sandbox: buildDir: yarn-project projectDir: yarn-project/aztec-sandbox dependencies: - yarn-project-prod - multiarch: true + multiarch: buildx aztec-faucet: buildDir: yarn-project @@ -153,7 +154,7 @@ cli: projectDir: yarn-project/cli dependencies: - yarn-project-prod - multiarch: true + multiarch: buildx boxes-blank-react: buildDir: yarn-project diff --git a/cspell.json b/cspell.json index 35d7779e39d..3d7775f13b2 100644 --- a/cspell.json +++ b/cspell.json @@ -21,6 +21,7 @@ "Bufferable", "bufs", "buildkit", + "buildx", "bytecodes", "calldatacopy", "callstack", @@ -93,6 +94,7 @@ "mplex", "msgpack", "muldiv", + "multiarch", "multivalue", "muxers", "Nargo", From 55df33dd37119a4bf5ad40bc7ed2783c5f81fbbb Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 16:00:26 +0000 Subject: [PATCH 21/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] --- build-system/scripts/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index 9a64d153e5e..17fa621dbe9 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -111,7 +111,7 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do # Substitute references to parent repo, with the relevent built image uri. DEPLOY_URI=$ECR_DEPLOY_URL/$PARENT_REPO PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:$PARENT_IMAGE_TAG - sed -i "s|^FROM $DEPLOY_URI|FROM $PARENT_IMAGE_URI|" $DOCKERFILE + sed -i "s|^FROM $DEPLOY_URI\([[:space:]]\|$\)|FROM $PARENT_IMAGE_URI|" $DOCKERFILE done COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) From 8c42a0c17dc3e658f25ea2bc932bdba5dc90fdf9 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 17:02:46 +0000 Subject: [PATCH 22/37] Lets play with buildx [ci rebuild aztec-sandbox] [ci rebuild cli] --- build-system/scripts/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index 17fa621dbe9..6c8169bccd0 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -111,7 +111,7 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do # Substitute references to parent repo, with the relevent built image uri. DEPLOY_URI=$ECR_DEPLOY_URL/$PARENT_REPO PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:$PARENT_IMAGE_TAG - sed -i "s|^FROM $DEPLOY_URI\([[:space:]]\|$\)|FROM $PARENT_IMAGE_URI|" $DOCKERFILE + awk '{if ($1 == "FROM" && $2 == "'$DEPLOY_URI'") $2 = "'$PARENT_IMAGE_URI'"; print $0}' $DOCKERFILE > _temp && mv _temp $DOCKERFILE done COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) From d717cf1644641c6ec2e6fcc0b497f88f7bc79c80 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 17:30:29 +0000 Subject: [PATCH 23/37] [ci force-deploy] From 5a699f19f0a6e9eeca3af7529d64a2c62aaf57c9 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 20:10:55 +0000 Subject: [PATCH 24/37] [ci force-deploy] --- .circleci/config.yml | 6 ++--- build-system/scripts/deploy_dockerhub | 35 +++++++-------------------- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 541f4979d2c..6368cbe7f68 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -974,9 +974,9 @@ jobs: name: "Deploy to dockerhub" command: | should_deploy || exit 0 - deploy_dockerhub noir x86_64,arm64 - deploy_dockerhub aztec-sandbox x86_64,arm64 - deploy_dockerhub cli x86_64,arm64 + deploy_dockerhub noir + deploy_dockerhub aztec-sandbox + deploy_dockerhub cli # Repeatable config for defining the workflow below. defaults: &defaults diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 047980966f5..31a5d6feafb 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -3,54 +3,37 @@ set -eu REPOSITORY=$1 -# TODO: Why even provide this? We can just figure it out by probing. -ARCH_LIST=$2 -function docker_or_dryrun { +function cmd_or_dryrun { if [ "$DRY_DEPLOY" -eq 1 ]; then - echo DRY RUN: docker $@ + echo DRY RUN: $@ else - retry docker $@ + retry $@ fi } echo "Repo: $REPOSITORY" echo "Arch List: $ARCH_LIST" +# Set tag to the commit tag (a version number) if we have one, otherwise branch name normalised to a tag format. if [ -n "$COMMIT_TAG" ]; then TAG=$(extract_tag_version $REPOSITORY true) else TAG=$(echo "$BRANCH" | sed 's/[^a-zA-Z0-9_.-]/_/g') fi -MANIFEST_DEPLOY_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$TAG +SRC_URI=$(calculate_image_uri $REPOSITORY) +DST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$TAG # Login to dockerhub and ecr dockerhub_login ecr_login -IFS=',' -for ARCH in $ARCH_LIST; do - IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY $ARCH) - retry docker pull $IMAGE_COMMIT_URI - - # Retag and push image. - IMAGE_DEPLOY_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$TAG-$ARCH - docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_URI - docker_or_dryrun push $IMAGE_DEPLOY_URI - - echo "Adding image $IMAGE_DEPLOY_URI to manifest $MANIFEST_DEPLOY_URI..." - docker_or_dryrun manifest create $MANIFEST_DEPLOY_URI --amend $IMAGE_DEPLOY_URI -done - -echo "Pushing manifest $MANIFEST_DEPLOY_URI..." -docker_or_dryrun manifest push --purge $MANIFEST_DEPLOY_URI +retry wget https://github.com/lework/skopeo-binary/releases/download/v1.13.3/skopeo-linux-amd64 +cmd_or_dryrun ./skopeo-linux-amd64 copy --all docker://$SRC_URI docker://$DST_URI if [ -n "$COMMIT_TAG" ]; then # Publish as latest. LATEST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:latest - echo "Tagging $MANIFEST_DEPLOY_URI as $LATEST_URI..." - docker_or_dryrun pull $MANIFEST_DEPLOY_URI - docker_or_dryrun tag $MANIFEST_DEPLOY_URI $LATEST_URI - docker_or_dryrun push $LATEST_URI + cmd_or_dryrun ./skopeo-linux-amd64 copy --all docker://$DST_URI docker://$LATEST_URI fi From a65b6d315e1613e952484d488857309f1c56c82e Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 20:16:28 +0000 Subject: [PATCH 25/37] [ci force-deploy] --- build-system/scripts/deploy_dockerhub | 1 - 1 file changed, 1 deletion(-) diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 31a5d6feafb..a3fe101607a 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -13,7 +13,6 @@ function cmd_or_dryrun { } echo "Repo: $REPOSITORY" -echo "Arch List: $ARCH_LIST" # Set tag to the commit tag (a version number) if we have one, otherwise branch name normalised to a tag format. if [ -n "$COMMIT_TAG" ]; then From 79767ff8e0a93eb2181d65db36db40048925059f Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 20:23:59 +0000 Subject: [PATCH 26/37] [ci force-deploy] --- build-system/scripts/deploy_dockerhub | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index a3fe101607a..761db5a823f 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -29,10 +29,11 @@ dockerhub_login ecr_login retry wget https://github.com/lework/skopeo-binary/releases/download/v1.13.3/skopeo-linux-amd64 -cmd_or_dryrun ./skopeo-linux-amd64 copy --all docker://$SRC_URI docker://$DST_URI +mv ./skopeo-linux-amd64 ./skopeo && chmod +x ./skopeo +cmd_or_dryrun ./skopeo copy --all docker://$SRC_URI docker://$DST_URI if [ -n "$COMMIT_TAG" ]; then # Publish as latest. LATEST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:latest - cmd_or_dryrun ./skopeo-linux-amd64 copy --all docker://$DST_URI docker://$LATEST_URI + cmd_or_dryrun ./skopeo copy --all docker://$DST_URI docker://$LATEST_URI fi From 9eed4a32ff248a8f327d494f86b0e12e9d47f8fc Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 20:44:31 +0000 Subject: [PATCH 27/37] [ci force-deploy] --- .circleci/config.yml | 55 --------------------------- build-system/scripts/build | 5 ++- build-system/scripts/deploy_dockerhub | 9 +++-- 3 files changed, 9 insertions(+), 60 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6368cbe7f68..409b90f4edd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -390,21 +390,6 @@ jobs: name: "Build and test" command: build aztec-sandbox - # aztec-sandbox-arm64: - # machine: - # image: ubuntu-2204:2023.07.2 - # resource_class: arm.large - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Build and test" - # # We need to force not to use docker buildkit because for some reason on arm only, it ends up making a call - # # out to eu-west2 despite the image being locally tagged, resulting in unauthorized 401. Weird docker bug? - # command: | - # echo "export DOCKER_BUILDKIT=" > $BASH_ENV - # build aztec-sandbox - cli: machine: image: ubuntu-2204:2023.07.2 @@ -416,21 +401,6 @@ jobs: name: "Build and test" command: build cli - # cli-arm64: - # machine: - # image: ubuntu-2204:2023.07.2 - # resource_class: arm.large - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Build and test" - # # We need to force not to use docker buildkit because for some reason on arm only, it ends up making a call - # # out to eu-west2 despite the image being locally tagged, resulting in unauthorized 401. Weird docker bug? - # command: | - # echo "export DOCKER_BUILDKIT=" > $BASH_ENV - # build cli - mainnet-fork: machine: image: ubuntu-2204:2023.07.2 @@ -453,19 +423,6 @@ jobs: name: "Build and test" command: build aztec-faucet | add_timestamps - # ecr-manifest: - # machine: - # image: ubuntu-2204:2023.07.2 - # resource_class: large - # steps: - # - *checkout - # - *setup_env - # - run: - # name: "Create ECR manifest" - # command: | - # create_ecr_manifest aztec-sandbox x86_64,arm64 - # create_ecr_manifest cli x86_64,arm64 - boxes-blank-react: machine: image: ubuntu-2204:2023.07.2 @@ -1095,18 +1052,6 @@ workflows: - aztec-sandbox: *defaults_yarn_project_prod - cli: *defaults_yarn_project_prod - aztec-faucet: *defaults_yarn_project_prod - # - aztec-sandbox-x86_64: *defaults_yarn_project_prod - # - aztec-sandbox-arm64: *defaults_yarn_project_prod - # - cli-x86_64: *defaults_yarn_project_prod - # - cli-arm64: *defaults_yarn_project_prod - # - aztec-faucet: *defaults_yarn_project_prod - # - ecr-manifest: - # requires: - # - aztec-sandbox-x86_64 - # - aztec-sandbox-arm64 - # - cli-x86_64 - # - cli-arm64 - # <<: *defaults # Boxes. - boxes-blank-react: diff --git a/build-system/scripts/build b/build-system/scripts/build index 6c8169bccd0..6906a275149 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -126,13 +126,14 @@ MULTIARCH=$(query_manifest multiarch $REPOSITORY) if [ "$MULTIARCH" == "buildx" ]; then # We've requested to use buildx. This will build both arch containers on the host machine using virtualization. # The result is a single image tag that supports multiarch. + # This is the simplest approach for build jobs that are not too intensive. docker buildx create --name builder --use docker buildx inspect --bootstrap docker buildx build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH --platform linux/amd64,linux/arm64 . --push else - # If multiarch is set to "host", the assumption is that we doing multiple builds on different architcture machines + # If multiarch is set to "host", the assumption is that we're doing multiple builds on different machine architectures # in parallel, and that there is a another job that runs afterwards to combine them into a manifest. - # In this case we need to augment the image tag with this host architecture to ensure it's uniqueness. + # In this case we need to augment the image tag with the hosts architecture to ensure its uniqueness. if [ "$MULTIARCH" == "host" ]; then IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY host) fi diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 761db5a823f..6561b471c57 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -28,12 +28,15 @@ DST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:$TAG dockerhub_login ecr_login +# Install skopeo, and immediately hack it to newer version. +sudo apt install -y skopeo retry wget https://github.com/lework/skopeo-binary/releases/download/v1.13.3/skopeo-linux-amd64 -mv ./skopeo-linux-amd64 ./skopeo && chmod +x ./skopeo -cmd_or_dryrun ./skopeo copy --all docker://$SRC_URI docker://$DST_URI +chmod +x ./skopeo-linux-amd64 && sudo mv ./skopeo-linux-amd64 /usr/bin/skopeo + +cmd_or_dryrun skopeo copy --all docker://$SRC_URI docker://$DST_URI if [ -n "$COMMIT_TAG" ]; then # Publish as latest. LATEST_URI=$DOCKERHUB_ACCOUNT/$REPOSITORY:latest - cmd_or_dryrun ./skopeo copy --all docker://$DST_URI docker://$LATEST_URI + cmd_or_dryrun skopeo copy --all docker://$DST_URI docker://$LATEST_URI fi From bcfb4806f34128b5b11ffc6a6ad2267e7cb6374e Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 21:06:33 +0000 Subject: [PATCH 28/37] [ci rebuild aztec-sandbox] [ci force-deploy] From 2cd5000348dff1b0b0d43c06b9ddc3f0e8682230 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 22:26:49 +0000 Subject: [PATCH 29/37] [ci force-release] From f0c0806489c364c5cc8c68d1385b98c253fe6742 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 22:31:24 +0000 Subject: [PATCH 30/37] [ci force-release] --- build-system/scripts/setup_env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index a108e8a43f7..601c4aa9f51 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -19,6 +19,7 @@ BASH_ENV=${BASH_ENV:-} BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) PROJECT=$(cat PROJECT) COMMIT_MESSAGE=$(git log -n 1 --pretty=format:"%s" $COMMIT_HASH) +PATH=$PATH:$BUILD_SYSTEM_PATH/scripts echo "COMMIT_HASH=$COMMIT_HASH" echo "COMMIT_TAG=$COMMIT_TAG" @@ -53,7 +54,7 @@ fi # - The deploy tag (used in api paths, subdomains), is a concatenation of the project name and deploy environment, # e.g. aztec-dev, aztec-testnet, or aztec-prod -if [ -n "$DEPLOY_ENV" ]; then +if [ -n "${DEPLOY_ENV:-}" ]; then DEPLOY_TAG=$PROJECT-$DEPLOY_ENV fi From c0fda33f3cc73578edd3af55091fcff190c7daaa Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 22:35:08 +0000 Subject: [PATCH 31/37] [ci force-release] --- build-system/scripts/setup_env | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 601c4aa9f51..954c987ad14 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -21,6 +21,8 @@ PROJECT=$(cat PROJECT) COMMIT_MESSAGE=$(git log -n 1 --pretty=format:"%s" $COMMIT_HASH) PATH=$PATH:$BUILD_SYSTEM_PATH/scripts +export BRANCH + echo "COMMIT_HASH=$COMMIT_HASH" echo "COMMIT_TAG=$COMMIT_TAG" echo "JOB_NAME=$JOB_NAME" @@ -82,8 +84,8 @@ echo export COMMIT_TAG=$COMMIT_TAG >> $BASH_ENV echo "export COMMIT_MESSAGE='${COMMIT_MESSAGE//\'/\'\\\'\'}'" >> $BASH_ENV echo export JOB_NAME=$JOB_NAME >> $BASH_ENV echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV -echo export DEPLOY_ENV=$DEPLOY_ENV >> $BASH_ENV -echo export DEPLOY_TAG=${DEPLOY_TAG:-} >> $BASH_ENV +echo export DEPLOY_ENV=${DEPLOY_ENV:-} >> $BASH_ENV +echo export DEPLOY_TAG=$DEPLOY_TAG >> $BASH_ENV echo export BRANCH=$BRANCH >> $BASH_ENV echo export PULL_REQUEST=$PULL_REQUEST >> $BASH_ENV echo export DRY_DEPLOY=${DRY_DEPLOY:-0} >> $BASH_ENV From c89fdef5da69710dbb53f06b638b61dd68e6d647 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 22:38:26 +0000 Subject: [PATCH 32/37] [ci force-release] --- build-system/scripts/setup_env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 954c987ad14..16763d1df5c 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -85,7 +85,7 @@ echo "export COMMIT_MESSAGE='${COMMIT_MESSAGE//\'/\'\\\'\'}'" >> $BASH_ENV echo export JOB_NAME=$JOB_NAME >> $BASH_ENV echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV echo export DEPLOY_ENV=${DEPLOY_ENV:-} >> $BASH_ENV -echo export DEPLOY_TAG=$DEPLOY_TAG >> $BASH_ENV +echo export DEPLOY_TAG=${DEPLOY_TAG:-} >> $BASH_ENV echo export BRANCH=$BRANCH >> $BASH_ENV echo export PULL_REQUEST=$PULL_REQUEST >> $BASH_ENV echo export DRY_DEPLOY=${DRY_DEPLOY:-0} >> $BASH_ENV From b0da8f591277b19526e178b79e529b66bae876e0 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 1 Dec 2023 22:50:53 +0000 Subject: [PATCH 33/37] [ci force-release] --- build-system/scripts/cond_spot_run_build | 2 +- build-system/scripts/cond_spot_run_script | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 6e970387cce..3333e5dec7c 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -6,4 +6,4 @@ REPOSITORY=$1 CPUS=$2 ARCH=${3:-x86_64} -cond_spot_run_script $REPOSITORY $CPUS $ARCH build $REPOSITORY $ARCH | add_timestamps +cond_spot_run_script $REPOSITORY $CPUS $ARCH build $REPOSITORY | add_timestamps diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 89d41c9af0e..4b1832127ac 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -19,8 +19,7 @@ CPUS=$2 ARCH=$3 shift 3 -# If the CPUS have a specific architecture assigned, we need to use that to build the success tag. -BASE_TAG=$(calculate_image_tag $REPOSITORY $ARCH) +BASE_TAG=$(calculate_image_tag $REPOSITORY) SUCCESS_TAG=$BASE_TAG if [ -n "${TAG_POSTFIX:-}" ]; then From b568069d860962ad4f3d15c7c8a8d885de7cc4f5 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sat, 2 Dec 2023 21:44:46 +0000 Subject: [PATCH 34/37] build_local fix. --- build-system/scripts/build_local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index b4572cecee0..7ef1d3c5d8e 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -98,7 +98,7 @@ for E in "${PROJECTS[@]}"; do echo -e "${YELLOW}Project or dependency has local modifications! Building...${RESET}" docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $DEPLOY_IMAGE_URI . else - if [ -z "$NO_CACHE" ] && docker image ls --format "{{.Repository}}:{{.Tag}}" | grep -q -w $CACHE_IMAGE_URI; then + if [ -z "$NO_CACHE" ] && docker image ls --format "{{.Repository}}:{{.Tag}}" | grep -q -w "$CACHE_IMAGE_URI$"; then echo -e "${GREEN}Image exists locally. Tagging as $DEPLOY_IMAGE_URI${RESET}" docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI else From 784c1f37f536bcbf269f68b4d26197afb0bbd549 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 4 Dec 2023 11:05:52 +0000 Subject: [PATCH 35/37] [ci rebuild] From 3908514c817d43c77e8dbae67b0b3c2498394cda Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 4 Dec 2023 12:15:31 +0000 Subject: [PATCH 36/37] maybe fix. --- yarn-project/noir-contracts/scripts/types.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/noir-contracts/scripts/types.sh b/yarn-project/noir-contracts/scripts/types.sh index 2747a38ef9b..abedef70d12 100755 --- a/yarn-project/noir-contracts/scripts/types.sh +++ b/yarn-project/noir-contracts/scripts/types.sh @@ -44,10 +44,10 @@ process() { CONTRACT=$1 cd $ROOT - NODE_OPTIONS=--no-warnings yarn ts-node --esm src/scripts/copy_source.ts $CONTRACT_NAME + node --no-warnings --loader ts-node/esm src/scripts/copy_source.ts $CONTRACT_NAME echo "Creating types for $CONTRACT" - NODE_OPTIONS=--no-warnings yarn ts-node --esm src/scripts/copy_output.ts $CONTRACT_NAME + node --no-warnings --loader ts-node/esm src/scripts/copy_output.ts $CONTRACT_NAME } format(){ From 307c47cc0c16ebfcd1c6381da70a4f3a607a1f85 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 4 Dec 2023 12:52:25 +0000 Subject: [PATCH 37/37] [ci force-release]