diff --git a/.circleci/config.yml b/.circleci/config.yml index ac54d75ec06b..b2531ae99b79 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -431,42 +431,6 @@ jobs: command: build aztec aztec_manifest_key: aztec - boxes: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small - steps: - - *checkout - - *setup_env - - run: - name: "Build" - command: cond_spot_run_build boxes 4 - aztec_manifest_key: boxes - - boxes-vanilla: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small - steps: - - *checkout - - *setup_env - - run: - name: "Test" - command: cond_spot_run_compose boxes 4 ./docker-compose.yml BOX=vanilla - aztec_manifest_key: boxes - - boxes-react: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small - steps: - - *checkout - - *setup_env - - run: - name: "Test" - command: cond_spot_run_compose boxes 4 ./docker-compose.yml BOX=react - aztec_manifest_key: boxes - end-to-end: machine: image: default @@ -754,20 +718,6 @@ workflows: # Artifacts - aztec-package: *defaults_yarn_project - # Boxes. - - boxes: - requires: - - aztec-package - <<: *defaults - - boxes-vanilla: - requires: - - boxes - <<: *defaults - - boxes-react: - requires: - - boxes - <<: *defaults - # End to end tests. - e2e-join: requires: @@ -788,8 +738,6 @@ workflows: - barretenberg-acir-tests-bb-sol - barretenberg-docs - build-docs - - boxes-vanilla - - boxes-react - noir-packages-tests - prover-client-test - e2e-join diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73dc79e18afd..a64abcd3d165 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,10 +207,52 @@ jobs: timeout-minutes: 15 run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache + boxes: + needs: build + runs-on: ${{ inputs.username || github.actor }}-x86 + steps: + - { + uses: actions/checkout@v4, + with: { ref: "${{ github.event.pull_request.head.sha }}" }, + } + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + # must be globally unique for build x runner + concurrency_key: boxes-${{ inputs.username || github.actor }}-x86-${{ matrix.test }} + - name: Build + working-directory: ./boxes + timeout-minutes: 10 + run: earthly-ci +export-boxes + + boxes-test: + needs: boxes + runs-on: ${{ inputs.username || github.actor }}-x86 + strategy: + fail-fast: false + matrix: + box: [vanilla, react] + browser: [chromium, webkit, firefox] + steps: + - { + uses: actions/checkout@v4, + with: { ref: "${{ github.event.pull_request.head.sha }}" }, + } + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + # must be globally unique for build x runner + concurrency_key: boxes-${{ inputs.username || github.actor }}-x86-${{ matrix.test }} + - name: Box test + working-directory: ./boxes + timeout-minutes: 10 + run: earthly-ci -P --no-output +test --box=${{ matrix.box }} --browser=${{ matrix.browser }} --mode=cache + merge-check: runs-on: ubuntu-latest needs: - e2e + - boxes-test - bb-native-tests - bb-bench - yarn-project-formatting diff --git a/boxes/.earthlyignore b/boxes/.earthlyignore new file mode 100644 index 000000000000..a2c3635374ab --- /dev/null +++ b/boxes/.earthlyignore @@ -0,0 +1,46 @@ +# Must include the .gitignore for all child projects as this is used by Earthly + +# Note due to how we use Eartlhy each .gitignore MUST accompany any earthfile that might actually copy these artifacts + +**/Earthfile +**/Readme.md +**/README.md +**/Dockerfile* +\*\*/docker-compose*.yml + +# root .gitignore contents + +dest +node_modules +.cache +scripts/.earthly +.pnp.cjs +.pnp.loader.mjs +build/ +.idea +cmake-build-debug +.terraform\* +.bootstrapped +.tsbuildinfo + +.graphite\* +.DS_Store + +\*_/_.dockerignore + +# .gitignore contents: + +.yarn/\* +!.yarn/releases + +node_modules +dist +artifacts +boxes/_/src/contracts/target +boxes/_/src/contracts/log + +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ +codegenCache.json diff --git a/boxes/Earthfile b/boxes/Earthfile index d790d1b0da5e..4ce2102ec88f 100644 --- a/boxes/Earthfile +++ b/boxes/Earthfile @@ -1,18 +1,130 @@ VERSION 0.8 -# Produces a container that can be run to test a specific box. See docker-compose.yml. -build: - # We need yarn. Start fresh container. +deps: + LOCALLY + LET packages = $(git ls-files "**/package*.json" package*.json) + LET tsconfigs = $(git ls-files "**/tsconfig*.json" tsconfig*.json) FROM node:18.19.0 - RUN apt update && apt install netcat-openbsd - COPY ../yarn-project+build/build /usr/src + + # copy the aztec yarn workspace, needed to resolve portal dependencies + COPY ../yarn-project+workspace/usr/src /usr/src + + WORKDIR /usr/src/boxes + + COPY --dir .yarn .yarnrc.yml yarn.lock . + FOR file IN $packages + COPY $file $file + END + RUN yarn install --immutable + RUN yarn install-browsers + FOR file IN $tsconfigs + COPY $file $file + END + +build: + FROM +deps + COPY ../noir/+nargo/nargo /usr/src/noir/noir-repo/target/release/nargo COPY ../noir-projects/+build/aztec-nr /usr/src/noir-projects/aztec-nr COPY ../noir-projects/+build/noir-protocol-circuits/crates/types /usr/src/noir-projects/noir-protocol-circuits/crates/types + WORKDIR /usr/src/boxes - COPY . . + ENV AZTEC_NARGO=/usr/src/noir/noir-repo/target/release/nargo ENV AZTEC_BUILDER=/usr/src/yarn-project/builder/aztec-builder-dest - RUN yarn && yarn build - RUN npx -y playwright@1.42 install --with-deps + COPY . . + RUN yarn build + +boxes: + FROM +build ENTRYPOINT ["/bin/sh", "-c"] + +BOX_TEST_LOCAL: + FUNCTION + ARG box + ARG browser + ARG compose_file=./docker-compose.yml + ARG debug="aztec:*" + LOCALLY + ENV BOX=$box + ENV BROWSER=$browser + ENV DEBUG=$debug + ENV CI=1 + WITH DOCKER \ + --load aztecprotocol/aztec:latest=../yarn-project/+aztec \ + --load aztecprotocol/boxes:latest=+boxes + RUN docker compose -f $compose_file up --exit-code-from=boxes --force-recreate + END + +# run locally and take from cache, used for our mainly x86 jobs +BOX_TEST_FROM_CACHE: + FUNCTION + ARG box + ARG browser + ARG compose_file=./docker-compose.yml + ARG debug="aztec:*" + ARG aztec_docker_tag + LOCALLY + ENV BOX=$box + ENV BROWSER=$browser + ENV DEBUG=$debug + ENV AZTEC_DOCKER_TAG=$aztec_docker_tag + ENV CI=1 + # need a different project name for each to run in parallel + LET project_name=$box_$browser + IF docker compose > /dev/null 2>&1 + LET CMD="docker compose" + ELSE + LET CMD="docker-compose" + END + # In CI we do not use WITH DOCKER as we have had issues with earthly copying big images + RUN $CMD -p $project_name -f $compose_file up --exit-code-from=boxes --force-recreate + +BOX_TEST_FROM_BUILD: + FUNCTION + ARG box + ARG browser + ARG compose_file=./docker-compose.yml + ARG debug="aztec:*" + FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + ENV BOX=$box + ENV BROWSER=$browser + ENV DEBUG=$debug + ENV CI=1 + COPY $compose_file $compose_file + # For ARM, we do use WITH DOCKER as we don't have many e2e tests, but note BOX_TEST_FROM_CACHE + WITH DOCKER \ + --load aztecprotocol/aztec:latest=../yarn-project/+aztec \ + --load aztecprotocol/boxes:latest=+boxes + # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs + RUN docker compose -f $compose_file up --exit-code-from=boxes --force-recreate + END + +BOX_TEST: + FUNCTION + ARG box + ARG browser + ARG compose_file=./docker-compose.yml + ARG mode=local + ARG debug="aztec:*" + LOCALLY + IF [ $mode = local ] + DO +BOX_TEST_LOCAL --box=$box --browser=$browser --compose_file=$compose_file --debug=$debug + ELSE IF [ $mode = cache ] + DO +BOX_TEST_FROM_CACHE --box=$box --browser=$browser --aztec_docker_tag=$(git rev-parse HEAD) --compose_file=$compose_file --debug=$debug + ELSE + DO +BOX_TEST_FROM_BUILD --box=$box --browser=$browser --compose_file=$compose_file --debug=$debug + END + +test: + ARG box=vanilla + ARG browser=chromium + ARG mode=local + DO +BOX_TEST --box=$box --browser=$browser --mode=$mode + +# for use with mode=cache +export-boxes: + BUILD +boxes + ARG EARTHLY_GIT_HASH + FROM +boxes + SAVE IMAGE aztecprotocol/boxes:$EARTHLY_GIT_HASH diff --git a/boxes/boxes/react/package.json b/boxes/boxes/react/package.json index 8d121e38a9d2..6707358fed35 100644 --- a/boxes/boxes/react/package.json +++ b/boxes/boxes/react/package.json @@ -6,7 +6,7 @@ "type": "module", "main": "./dist/index.js", "scripts": { - "compile": "cd src/contracts && ${AZTEC_NARGO:-aztec-nargo} compile", + "compile": "cd src/contracts && ${AZTEC_NARGO:-aztec-nargo} compile --silence-warnings", "codegen": "${AZTEC_BUILDER:-aztec-builder} codegen src/contracts/target -o artifacts", "clean": "rm -rf ./dist .tsbuildinfo ./artifacts ./src/contracts/target", "prep": "yarn clean && yarn compile && yarn codegen", diff --git a/boxes/boxes/react/playwright.config.ts b/boxes/boxes/react/playwright.config.ts index 95e73847ebea..9b6a8f517fcc 100644 --- a/boxes/boxes/react/playwright.config.ts +++ b/boxes/boxes/react/playwright.config.ts @@ -25,11 +25,10 @@ export default defineConfig({ name: 'firefox', use: { ...devices['Desktop Firefox'] }, }, - - // { - // name: 'webkit', - // use: { ...devices['Desktop Safari'] }, - // }, + { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + }, ], webServer: { command: 'yarn serve', diff --git a/boxes/boxes/vanilla/package.json b/boxes/boxes/vanilla/package.json index f2d6ee850ee0..c029a78cde0f 100644 --- a/boxes/boxes/vanilla/package.json +++ b/boxes/boxes/vanilla/package.json @@ -5,7 +5,7 @@ "version": "0.1.0", "type": "module", "scripts": { - "compile": "cd src/contracts && ${AZTEC_NARGO:-aztec-nargo} compile", + "compile": "cd src/contracts && ${AZTEC_NARGO:-aztec-nargo} compile --silence-warnings", "codegen": "${AZTEC_BUILDER:-aztec-builder} codegen src/contracts/target -o artifacts", "clean": "rm -rf ./dest .tsbuildinfo ./artifacts ./src/contracts/target", "prep": "yarn clean && yarn compile && yarn codegen && tsc -b", diff --git a/boxes/boxes/vanilla/playwright.config.ts b/boxes/boxes/vanilla/playwright.config.ts index 96c5e3b073d3..9b6a8f517fcc 100644 --- a/boxes/boxes/vanilla/playwright.config.ts +++ b/boxes/boxes/vanilla/playwright.config.ts @@ -25,7 +25,6 @@ export default defineConfig({ name: 'firefox', use: { ...devices['Desktop Firefox'] }, }, - { name: 'webkit', use: { ...devices['Desktop Safari'] }, diff --git a/boxes/docker-compose.yml b/boxes/docker-compose.yml index 99c669cbfd1a..b21036bb9342 100644 --- a/boxes/docker-compose.yml +++ b/boxes/docker-compose.yml @@ -5,7 +5,7 @@ services: command: "'anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337'" aztec: - image: aztecprotocol/aztec + image: aztecprotocol/aztec:${AZTEC_DOCKER_TAG:-latest} environment: ETHEREUM_HOST: http://ethereum:8545 CHAIN_ID: 31337 @@ -15,13 +15,14 @@ services: WS_BLOCK_CHECK_INTERVAL_MS: 50 PXE_BLOCK_POLLING_INTERVAL_MS: 50 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 + depends_on: + - ethereum boxes: - image: aztecprotocol/boxes + image: aztecprotocol/boxes:${AZTEC_DOCKER_TAG:-latest} entrypoint: > sh -c ' - while ! nc -z aztec 8080; do sleep 1; done; - yarn workspace @aztec/$BOX test + yarn workspace @aztec/$BOX test --project=$BROWSER ' environment: DEBUG: "aztec:*" @@ -30,3 +31,8 @@ services: CHAIN_ID: 31337 PXE_URL: http://aztec:8080 BOX: ${BOX:-vanilla} + CI: ${CI:-} + BROWSER: ${BROWSER:-chromium} + depends_on: + aztec: + condition: service_healthy diff --git a/boxes/package.json b/boxes/package.json index bff118dbe5be..fb55d34e74d7 100644 --- a/boxes/package.json +++ b/boxes/package.json @@ -6,6 +6,7 @@ "scripts": { "compile": "yarn workspaces foreach -A -v run compile", "build": "yarn workspaces foreach -A -v run build", + "install-browsers": "playwright install --with-deps", "publish": "yarn npm publish" }, "workspaces": [ @@ -36,5 +37,8 @@ "pino": "^8.19.0", "pino-pretty": "^10.3.1", "tiged": "^2.12.6" + }, + "devDependencies": { + "@playwright/test": "1.42.0" } } diff --git a/boxes/yarn.lock b/boxes/yarn.lock index d1a4ff91e966..8c1567f2d704 100644 --- a/boxes/yarn.lock +++ b/boxes/yarn.lock @@ -3586,6 +3586,7 @@ __metadata: "@inquirer/confirm": "npm:^3.0.0" "@inquirer/input": "npm:^2.0.0" "@inquirer/select": "npm:^2.0.0" + "@playwright/test": "npm:1.42.0" axios: "npm:^1.6.7" commander: "npm:^12.0.0" node-pty: "npm:^1.0.0" diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 0821b63f8a15..f409ee08d298 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -47,6 +47,10 @@ build: ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true RUN ./bootstrap.sh full +workspace: + FROM +build + SAVE ARTIFACT /usr/src /usr/src + aztec-prod: FROM +build RUN yarn workspaces focus @aztec/aztec --production && yarn cache clean @@ -58,8 +62,14 @@ aztec-prod-slim: aztec: FROM +aztec-prod-slim + RUN apt update && apt install -y curl && rm -rf /var/lib/apt/lists/* && apt-get clean + LET port=8080 + ENV PORT=$port + HEALTHCHECK --interval=10s --timeout=10s --retries=6 \ + CMD curl -fsSd '{\"jsonrpc\":\"2.0\",\"method\":\"pxe_getNodeInfo\",\"id\":1}' http://127.0.0.1:$port ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec/dest/bin/index.js"] - EXPOSE 8080 + EXPOSE $port + SAVE IMAGE aztecprotocol/aztec:latest end-to-end-prod: FROM +build