From 097a888b1f60d285595dbae6ebac5af32f9ace67 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 26 Feb 2024 13:08:05 +0000 Subject: [PATCH 1/3] fix: AZTEC_PORT variable for devnet (#4700) - `AZTEC_NODE_PORT` -> `AZTEC_PORT` - Log Availability Oracle contract address on node start --- yarn-project/aztec-node/src/aztec-node/server.ts | 3 ++- yarn-project/aztec-node/terraform/main.tf | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 4d182caaa5ef..d31b95afce2c 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -87,7 +87,8 @@ export class AztecNodeService implements AztecNode { `Registry: ${config.l1Contracts.registryAddress.toString()}\n` + `Inbox: ${config.l1Contracts.inboxAddress.toString()}\n` + `Outbox: ${config.l1Contracts.outboxAddress.toString()}\n` + - `Contract Emitter: ${config.l1Contracts.contractDeploymentEmitterAddress.toString()}`; + `Contract Emitter: ${config.l1Contracts.contractDeploymentEmitterAddress.toString()}\n` + + `Availability Oracle: ${config.l1Contracts.availabilityOracleAddress.toString()}`; this.log(message); } diff --git a/yarn-project/aztec-node/terraform/main.tf b/yarn-project/aztec-node/terraform/main.tf index aa030c163d71..ce2e3b42bffc 100644 --- a/yarn-project/aztec-node/terraform/main.tf +++ b/yarn-project/aztec-node/terraform/main.tf @@ -180,7 +180,7 @@ resource "aws_ecs_task_definition" "aztec-node" { "value": "false" }, { - "name": "AZTEC_NODE_PORT", + "name": "AZTEC_PORT", "value": "80" }, { From c2027e3f58279fc9fa7c8e5c1b7fdcf832555d90 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 26 Feb 2024 13:16:41 +0000 Subject: [PATCH 2/3] fix: PXE devnet connectivity (#4759) - Turn on retries for PXE client talking to Aztec node - turn on `pxe_` namespacing by default when running PXE outside of the `aztec` binary context --- .../aztec.js/src/rpc_clients/pxe_client.ts | 22 +++++++++---------- yarn-project/end-to-end/src/fixtures/utils.ts | 3 ++- .../src/json-rpc/client/json_rpc_client.ts | 2 +- .../pxe/src/pxe_http/pxe_http_server.ts | 5 +++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts b/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts index 296daead28bc..61a7a480985c 100644 --- a/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts +++ b/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts @@ -32,28 +32,28 @@ import { createJsonRpcClient, makeFetch } from '@aztec/foundation/json-rpc/clien * @param fetch - The fetch implementation to use. * @returns A JSON-RPC client of PXE. */ -export const createPXEClient = (url: string, fetch = makeFetch([1, 2, 3], true)): PXE => +export const createPXEClient = (url: string, fetch = makeFetch([1, 2, 3], false)): PXE => createJsonRpcClient( url, { - CompleteAddress, - FunctionSelector, + AuthWitness, AztecAddress, - TxExecutionRequest, + CompleteAddress, ContractData, + FunctionSelector, + EthAddress, ExtendedContractData, + ExtendedNote, ExtendedUnencryptedL2Log, - TxHash, - EthAddress, - Point, Fr, GrumpkinScalar, - Note, - ExtendedNote, - AuthWitness, + L2Block, L2Tx, LogId, - L2Block, + Note, + Point, + TxExecutionRequest, + TxHash, }, { Tx, TxReceipt, L2BlockL2Logs }, false, diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 8ac596c5faa7..2edd58fb46ee 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -19,6 +19,7 @@ import { createDebugLogger, createPXEClient, deployL1Contracts, + makeFetch, waitForPXE, } from '@aztec/aztec.js'; import { @@ -164,7 +165,7 @@ async function setupWithRemoteEnvironment( logger(`Creating Aztec Node client to remote host ${aztecNodeUrl}`); const aztecNode = createAztecNodeClient(aztecNodeUrl); logger(`Creating PXE client to remote host ${PXE_URL}`); - const pxeClient = createPXEClient(PXE_URL); + const pxeClient = createPXEClient(PXE_URL, makeFetch([1, 2, 3], true)); await waitForPXE(pxeClient, logger); logger('JSON RPC client connected to PXE'); logger(`Retrieving contract addresses from ${PXE_URL}`); diff --git a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts index c62b20d66f73..1e0c69d25794 100644 --- a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts +++ b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts @@ -77,7 +77,7 @@ export function makeFetch(retries: number[], noRetry: boolean, log?: DebugLogger return async (host: string, rpcMethod: string, body: any, useApiEndpoints: boolean) => { return await retry( () => defaultFetch(host, rpcMethod, body, useApiEndpoints, noRetry), - `JsonRpcClient request to ${host}`, + `JsonRpcClient request ${rpcMethod} to ${host}`, makeBackoff(retries), log, true, diff --git a/yarn-project/pxe/src/pxe_http/pxe_http_server.ts b/yarn-project/pxe/src/pxe_http/pxe_http_server.ts index 300da326589c..1a379c9d0a99 100644 --- a/yarn-project/pxe/src/pxe_http/pxe_http_server.ts +++ b/yarn-project/pxe/src/pxe_http/pxe_http_server.ts @@ -20,7 +20,7 @@ import { FunctionSelector } from '@aztec/circuits.js'; import { AztecAddress } from '@aztec/foundation/aztec-address'; import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr, GrumpkinScalar, Point } from '@aztec/foundation/fields'; -import { JsonRpcServer } from '@aztec/foundation/json-rpc/server'; +import { JsonRpcServer, createNamespacedJsonRpcServer } from '@aztec/foundation/json-rpc/server'; import http from 'http'; @@ -63,7 +63,8 @@ export function createPXERpcServer(pxeService: PXE): JsonRpcServer { * @returns A running http server. */ export function startPXEHttpServer(pxeService: PXE, port: string | number): http.Server { - const rpcServer = createPXERpcServer(pxeService); + const pxeServer = createPXERpcServer(pxeService); + const rpcServer = createNamespacedJsonRpcServer([{ pxe: pxeServer }]); const app = rpcServer.getApp(); const httpServer = http.createServer(app.callback()); From f3c65ce75637bd47aca849a08b567b06a69318b0 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 14:06:09 +0000 Subject: [PATCH 3/3] chore: decouple ypb (#4749) This: * Decouples (partially) yarn-project-base build from rest of pipeline. Actually it's not quite what I hoped as I realised that at present to build ypb we need bb.js and noir packages to have been built first (which in turn require bb and noir). This maybe able to be worked around. My hope was that I fully decouple it so that we would only perform the expensive yarn install, if the package.json files had changed. Practically this partial decoupling probably won't improve CI times that much, but it still makes sense to do what's done here. * Fixes bug in boxes whereby it didn't wait for aztec to fully start before running test. * Remove `prep` call which compiles from `test`. Think this was added by @signorecello but running the container tests should not be doing any builds. --- .circleci/config.yml | 7 +++---- boxes/.dockerignore | 3 ++- boxes/Dockerfile | 3 ++- boxes/docker-compose.yml | 6 +++++- boxes/react/package.json | 2 +- build_manifest.yml | 8 +++----- yarn-project/Dockerfile | 15 ++++++++++----- yarn-project/yarn-project-base/Dockerfile | 20 ++------------------ 8 files changed, 28 insertions(+), 36 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eb276b39bdf7..435d9f88a99e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1389,16 +1389,15 @@ workflows: # Yarn Project - yarn-project-base: requires: - - avm-transpiler - - l1-contracts - bb-js - noir-packages - - noir-projects - - boxes-files <<: *defaults - yarn-project: requires: - yarn-project-base + - l1-contracts + - noir-projects + - boxes-files <<: *defaults - yarn-project-prod: *defaults_yarn_project - yarn-project-formatting: *defaults_yarn_project diff --git a/boxes/.dockerignore b/boxes/.dockerignore index bde85ad2dcda..7eab5f5a5cbb 100644 --- a/boxes/.dockerignore +++ b/boxes/.dockerignore @@ -5,4 +5,5 @@ node_modules .tsbuildinfo Dockerfile* .dockerignore -docker-compose.yml \ No newline at end of file +docker-compose.yml +**/artifacts \ No newline at end of file diff --git a/boxes/Dockerfile b/boxes/Dockerfile index 9241170b544c..ef381f2fff51 100644 --- a/boxes/Dockerfile +++ b/boxes/Dockerfile @@ -6,6 +6,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects # We need yarn. Start fresh container. FROM node:18.19.0 +RUN apt update && apt install netcat-openbsd COPY --from=aztec /usr/src /usr/src COPY --from=noir /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo COPY --from=noir-projects /usr/src/noir-projects/aztec-nr /usr/src/noir-projects/aztec-nr @@ -13,4 +14,4 @@ WORKDIR /usr/src/boxes ENV AZTEC_NARGO=/usr/src/noir/target/release/nargo ENV AZTEC_CLI=/usr/src/yarn-project/cli/aztec-cli-dest RUN yarn && yarn build -ENTRYPOINT ["yarn", "workspace"] +ENTRYPOINT ["/bin/sh", "-c"] diff --git a/boxes/docker-compose.yml b/boxes/docker-compose.yml index 957c8ee68528..18ab65d443fd 100644 --- a/boxes/docker-compose.yml +++ b/boxes/docker-compose.yml @@ -18,7 +18,11 @@ services: boxes: image: aztecprotocol/boxes - command: "@aztec/$BOX test" + entrypoint: > + sh -c ' + while ! nc -z aztec 8080; do sleep 1; done; + yarn workspace @aztec/$BOX test + ' environment: DEBUG: "aztec:*" DEBUG_COLORS: "true" diff --git a/boxes/react/package.json b/boxes/react/package.json index 1d859052a837..9a5969152b6a 100644 --- a/boxes/react/package.json +++ b/boxes/react/package.json @@ -14,7 +14,7 @@ "serve": "serve -p 3000 ./dist", "formatting": "prettier --check ./src && eslint ./src", "formatting:fix": "prettier -w ./src", - "test": "yarn prep && NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --runInBand" + "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --runInBand" }, "jest": { "preset": "ts-jest/presets/default-esm", diff --git a/build_manifest.yml b/build_manifest.yml index 665e16f6b240..7f29658c01e1 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -135,13 +135,8 @@ yarn-project-base: - ^yarn-project/yarn.lock - ^yarn-project/.*/package.json$ dependencies: - - l1-contracts - bb.js - - noir - noir-packages - - boxes-files - - avm-transpiler - - noir-projects yarn-project: buildDir: yarn-project @@ -151,6 +146,9 @@ yarn-project: - ^yarn-project/cli/aztec-cli dependencies: - yarn-project-base + - l1-contracts + - boxes-files + - noir-projects yarn-project-prod: buildDir: yarn-project diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index 9de7d8776b82..aea67d0c5886 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -4,19 +4,24 @@ # - Run the tests. # - Run the formatter checks. # Any subsequent build steps needed to support downstream containers should be done in those containers build files. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base as builder +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/boxes-files as boxes-files -# Copy in the entire workspace. +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base +COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts +COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects +COPY --from=boxes-files /usr/src/boxes /usr/src/boxes COPY . . - +# Generate L1 contract TypeScript artifacts. +RUN cd l1-artifacts && ./scripts/generate-artifacts.sh && rm -rf /usr/src/l1-contracts # This is actually our code generation tool. Needed to build contract typescript wrappers. RUN yarn workspace @aztec/noir-compiler build -# Builds noir contracts (TODO: move this stage pre yarn-project). Generates typescript wrappers. +# Generates typescript wrappers. RUN yarn workspace @aztec/noir-contracts.js build:contracts # We need to build accounts as it needs to copy in account contracts from noir-contracts. RUN yarn workspace @aztec/accounts build:copy-contracts RUN yarn workspace @aztec/protocol-contracts build:copy-contracts RUN yarn workspace @aztec/noir-protocol-circuits-types build RUN yarn tsc -b - ENTRYPOINT ["yarn"] diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index 933ec230be2a..b53ed652bc0b 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -42,25 +42,15 @@ # RUN yarn workspaces focus --production && yarn cache clean && rm -rf ../**/src # - Create final slim image by copying needed dirs into a fresh image. # -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js as bb.js -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-packages as noir-packages -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/boxes-files as boxes-files FROM node:18.19.0 RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean -# Copy L1 contracts. -COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts -# Copy in bb.js +# Copy in portalled packages. COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts -# Copy in noir packages COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages -# Copy in noir projects -COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects -# Copy in boxes -COPY --from=boxes-files /usr/src/boxes /usr/src/boxes # We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they # walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution @@ -88,11 +78,5 @@ RUN yarn --immutable && rm -rf /root/.cache/puppeteer && /bin/bash -c '\ [[ $F =~ (.*-) ]] && ln $F /root/.yarn/berry/cache/${BASH_REMATCH[1]}8.zip; \ done' -# If everything's worked properly, we should no longer need access to the network. -RUN echo "enableNetwork: false" >> .yarnrc.yml - # Check package.json inheritance and tsconfig project references. -RUN yarn prepare:check - -# Generate L1 contract TypeScript artifacts. -RUN cd l1-artifacts && ./scripts/generate-artifacts.sh && rm -rf /usr/src/l1-contracts \ No newline at end of file +RUN yarn prepare:check \ No newline at end of file