chore: BI build tweaks (#2487)

* Small bump to wasm build time by parallelising. * Add cond_spot_run_container script. * Tighter rebuild patterns on yarn-project to target code. * Newer version of yarn. * Just use perl in a script rather than doing os detection. * Remove unneeded Dockerfiles. * Add some missing dependencies detected by pnp spike. * Add a more generalised dockerignore for yarn-project-base build, but we can't use it until April when ubuntu 24 is out and we get docker 24. --------- Co-authored-by: ludamad <adam.domurad@gmail.com>
AztecProtocol · Sep 26, 2023 · f8b6548 · f8b6548
1 parent 36896e7
commit f8b6548
Show file tree

Hide file tree

Showing 32 changed files with 1,118 additions and 1,012 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -376,7 +376,7 @@ jobs:
   yarn-project-base:
     machine:
       image: ubuntu-2204:2023.07.2
-    resource_class: xlarge
+    resource_class: large
     steps:
       - *checkout
       - *setup_env
@@ -395,27 +395,27 @@ jobs:
           name: Build
           command: build yarn-project | add_timestamps
 
-  yarn-project-format:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
+  yarn-project-formatting:
+    machine:
+      image: ubuntu-2204:2023.07.2
+    resource_class: large
     steps:
       - *checkout
       - *setup_env
       - run:
-          name: Build
-          command: cond_spot_run_script yarn-project 32 cond_run_container yarn-project formatting | add_timestamps
+          name: Check Formatting
+          command: cond_run_container yarn-project formatting | add_timestamps
 
-  yarn-project-test:
+  yarn-project-tests:
     docker:
       - image: aztecprotocol/alpine-build-image
     resource_class: small
     steps:
       - *checkout
       - *setup_env
       - run:
-          name: Build
-          command: cond_spot_run_script yarn-project 64 cond_run_container yarn-project test | add_timestamps
+          name: Test
+          command: cond_spot_run_container yarn-project 64 test | add_timestamps
 
   aztec-sandbox-base:
     machine:
@@ -1130,18 +1130,18 @@ workflows:
             - circuits-wasm-linux-clang
             - l1-contracts
           <<: *defaults
-      - yarn-project-format:
+      - yarn-project:
           requires:
-            - yarn-project
+            - yarn-project-base
+            - noir-contracts-build
           <<: *defaults
-      - yarn-project-test:
+      - yarn-project-formatting:
           requires:
             - yarn-project
           <<: *defaults
-      - yarn-project:
+      - yarn-project-tests:
           requires:
-            - yarn-project-base
-            - noir-contracts-build
+            - yarn-project
           <<: *defaults
 
       - end-to-end: *yarn_project

diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.wasm-linux-clang b/barretenberg/cpp/dockerfiles/Dockerfile.wasm-linux-clang
@@ -4,8 +4,11 @@ WORKDIR /usr/src/barretenberg/cpp
 COPY ./scripts/install-wasi-sdk.sh ./scripts/install-wasi-sdk.sh
 RUN ./scripts/install-wasi-sdk.sh
 COPY . .
-RUN cmake --preset wasm && cmake --build --preset wasm
-RUN cmake --preset wasm-threads && cmake --build --preset wasm-threads
+# Building both wasm's in parallel reduces build from 120s to 80s.
+RUN (cmake --preset wasm && cmake --build --preset wasm) & \
+    (cmake --preset wasm-threads && cmake --build --preset wasm-threads) & \
+    wait
+
 
 FROM scratch
 WORKDIR /usr/src/barretenberg/cpp

diff --git a/bootstrap_docker.sh b/bootstrap_docker.sh
@@ -22,7 +22,6 @@
 set -e
 
 PROJECT_NAME=${1:-}
-COMMIT_HASH=$(git rev-parse HEAD)
 
 if [ -z "$PROJECT_NAME" ]; then
   echo "usage: $0 <project_name>"
@@ -31,7 +30,7 @@ fi
 
 cd "$(dirname "$0")"
 
-source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER > /dev/null
+source ./build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 build_local $PROJECT_NAME
 
 if [ -z "$PROJECT_NAME" ]; then

diff --git a/build-system/scripts/cond_spot_run_container b/build-system/scripts/cond_spot_run_container
@@ -0,0 +1,10 @@
+#!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+REPOSITORY=$1
+CPUS=$2
+shift 2
+
+export TAG_POSTFIX=$JOB_NAME
+cond_spot_run_script $REPOSITORY $CPUS cond_run_container $REPOSITORY $@
diff --git a/build_manifest.yml b/build_manifest.yml
@@ -98,8 +98,6 @@ noir-contracts-build:
   rebuildPatterns:
     - ^yarn-project/noir-contracts/
     - ^yarn-project/aztec-nr/
-  # Prevent automatic querying of get_dependencies.sh script. Needed because this looks like a yarn project.
-  dependencies: []
 
 yarn-project-base:
   buildDir: yarn-project
@@ -113,6 +111,9 @@ yarn-project-base:
 
 yarn-project:
   buildDir: yarn-project
+  rebuildPatterns:
+    - ^yarn-project/.*\\.(ts|js|cjs|mjs|json|html)$
+    - ^yarn-project/Dockerfile
   dependencies:
     - yarn-project-base
     - noir-contracts-build

diff --git a/yarn-project/.dockerignore b/yarn-project/.dockerignore
@@ -10,6 +10,8 @@
 **/*.tsbuildinfo
 **/Dockerfile*
 **/node_modules
+Dockerfile
 
 noir-contracts/src/types
-noir-contracts/src/artifacts
+noir-contracts/src/artifacts
+noir-contracts/target
diff --git a/yarn-project/.yarn/releases/yarn-3.4.1.cjs b/yarn-project/.yarn/releases/yarn-3.4.1.cjs
diff --git a/yarn-project/.yarn/releases/yarn-3.6.3.cjs b/yarn-project/.yarn/releases/yarn-3.6.3.cjs
diff --git a/yarn-project/.yarnrc.yml b/yarn-project/.yarnrc.yml
@@ -1,7 +1,7 @@
+nodeLinker: node-modules
+
 plugins:
   - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
     spec: '@yarnpkg/plugin-workspace-tools'
 
-# note that pnp is used in CI
-nodeLinker: node-modules
-yarnPath: .yarn/releases/yarn-3.4.1.cjs
+yarnPath: .yarn/releases/yarn-3.6.3.cjs
diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile
@@ -1,15 +1,22 @@
 # This base dockerfile adds all the remaining source files, performs artifact generation, and builds the project.
+# See yarn-project-base/Dockerfile for deeper insight into why things are how they are.
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-contracts-build as noir
-FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base as builder
 
 # Copy in the entire workspace.
 COPY . .
 
 # Generate Noir contract TypeScript artifacts.
 COPY --from=noir /usr/src/yarn-project/noir-contracts/target /usr/src/yarn-project/noir-contracts/target
 # Run yarn build to have the json ABIs available for the types generator, generate types, build again.
+RUN apk add perl
 RUN cd /usr/src/yarn-project/noir-contracts && yarn build && ./scripts/types_all.sh && yarn build
+# Cleanup to reduce final image size.
+RUN rm -rf noir-contracts/target
 
-# Build the entire project and check formatting.
+# Build the entire project.
 RUN yarn tsc -b
-ENTRYPOINT [ "yarn" ]
+
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base
+COPY --from=builder /usr/src/yarn-project /usr/src/yarn-project
+ENTRYPOINT ["yarn"]
diff --git a/yarn-project/Dockerfile.format b/yarn-project/Dockerfile.format
diff --git a/yarn-project/Dockerfile.test b/yarn-project/Dockerfile.test
diff --git a/yarn-project/acir-simulator/package.json b/yarn-project/acir-simulator/package.json
@@ -44,6 +44,8 @@
     "@jest/globals": "^29.5.0",
     "@rushstack/eslint-patch": "^1.1.4",
     "@types/jest": "^29.5.0",
+    "@types/levelup": "^5.1.3",
+    "@types/memdown": "^3.0.2",
     "@types/node": "^18.7.23",
     "jest": "^29.5.0",
     "jest-mock-extended": "^3.0.4",

diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json
@@ -45,6 +45,8 @@
     "@aztec/world-state": "workspace:^",
     "koa": "^2.14.2",
     "koa-router": "^12.0.0",
+    "levelup": "^5.1.1",
+    "memdown": "^6.1.1",
     "tslib": "^2.4.0"
   },
   "devDependencies": {

diff --git a/yarn-project/aztec-sandbox/Dockerfile b/yarn-project/aztec-sandbox/Dockerfile
@@ -2,10 +2,6 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder
 
 ARG COMMIT_TAG=""
 
-# Remove SRS files (currently not producing proofs)
-RUN rm -rf /usr/src/barretenberg/cpp/srs_db/ignition/monomial
-COPY . .
-
 # Update pxe version if COMMIT_TAG has been used
 WORKDIR /usr/src/yarn-project/pxe
 RUN if [[ -n "${COMMIT_TAG}" ]]; then \
@@ -17,17 +13,13 @@ RUN if [[ -n "${COMMIT_TAG}" ]]; then \
   jq --arg v ${COMMIT_TAG} '.version = $v' package.json > _temp && mv _temp package.json; \
   fi
 
-RUN yarn build && yarn formatting && yarn test
-
-# Prune dev dependencies. See comment in base image.
-RUN yarn cache clean
-RUN yarn workspaces focus --production > /dev/null
+# Productionify. See comment in yarn-project-base/Dockerfile.
+RUN yarn cache clean && yarn workspaces focus --production
 
+# Create final, minimal size image.
 FROM node:18-alpine
-
 COPY --from=builder /usr/src/ /usr/src/
 WORKDIR /usr/src/yarn-project/aztec-sandbox
-
 ENTRYPOINT ["yarn"]
 CMD [ "start" ]
 EXPOSE 8080
diff --git a/yarn-project/aztec.js/webpack.config.js b/yarn-project/aztec.js/webpack.config.js
@@ -63,11 +63,9 @@ export default {
       path: false,
       url: false,
       worker_threads: false,
-      events: require.resolve('events/'),
       buffer: require.resolve('buffer/'),
       util: require.resolve('util/'),
       stream: require.resolve('stream-browserify'),
-      string_decoder: require.resolve('string_decoder/'),
       tty: require.resolve('tty-browserify'),
     },
   },

diff --git a/yarn-project/boxes/blank-react/src/scripts/call_contract_function.ts b/yarn-project/boxes/blank-react/src/scripts/call_contract_function.ts
@@ -20,5 +20,4 @@ export async function callContractFunction(
   return contract.methods[functionName](...typedArgs)
     .send()
     .wait();
-
 }
diff --git a/yarn-project/boxes/blank/src/index.ts b/yarn-project/boxes/blank/src/index.ts
@@ -70,7 +70,6 @@ export async function handleInteractClick(contractAddress: string) {
   );
 }
 
-
 export const getFunctionAbi = (contractAbi: any, functionName: string) => {
   const functionAbi = contractAbi.functions.find((f: FunctionAbi) => f.name === functionName);
   if (!functionAbi) throw new Error(`Function ${functionName} not found in abi`);

diff --git a/yarn-project/canary/Dockerfile b/yarn-project/canary/Dockerfile
@@ -1,22 +1,17 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder
 ARG COMMIT_TAG="."
 
-COPY . .
-
 # Setup & Build end-to-end package
 WORKDIR /usr/src/yarn-project/end-to-end
-
 # Extract canary @aztec dependencies
 RUN ../canary/scripts/extract_packages.sh ../canary/package.json > ./target_pkgs.txt
 # Update end-to-end dependencies with target version
 RUN ./scripts/setup_canary.sh "$COMMIT_TAG" ./target_pkgs.txt
 RUN rm ./target_pkgs.txt
-RUN yarn && yarn build
 
 # Build canary package
 WORKDIR /usr/src/yarn-project/canary
 RUN ./scripts/update_packages.sh "$COMMIT_TAG"
-RUN yarn && yarn build
 
 FROM node:18-alpine
 RUN apk update && apk add --no-cache udev ttf-freefont chromium curl jq bash

diff --git a/yarn-project/canary/Dockerfile.build b/yarn-project/canary/Dockerfile.build
@@ -1,13 +1,7 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder
-
-COPY . .
-
 WORKDIR /usr/src/yarn-project/canary
-RUN yarn build
-
-# Prune dev dependencies. See comment in base image.
-RUN yarn cache clean
-RUN yarn workspaces focus --production > /dev/null
+# Productionify. See comment in yarn-project-base/Dockerfile.
+RUN yarn cache clean && yarn workspaces focus --production
 
 FROM node:18-alpine
 COPY --from=builder /usr/src /usr/src

diff --git a/yarn-project/canary/package.json b/yarn-project/canary/package.json
@@ -30,6 +30,7 @@
     "@types/jest": "^29.5.0",
     "@types/koa-static": "^4.0.2",
     "@types/node": "^18.7.23",
+    "abitype": "^0.9.8",
     "jest": "^29.5.0",
     "koa": "^2.14.2",
     "koa-static": "^5.0.0",

diff --git a/yarn-project/canary/src/utils.ts b/yarn-project/canary/src/utils.ts
@@ -83,7 +83,7 @@ export async function deployL1Contract(
     abi,
     bytecode,
     args,
-  });
+  } as any);
 
   const receipt = await publicClient.waitForTransactionReceipt({ hash });
   const contractAddress = receipt.contractAddress;

diff --git a/yarn-project/cli/package.json b/yarn-project/cli/package.json
@@ -56,6 +56,7 @@
     "@types/jest": "^29.5.0",
     "@types/lodash.startcase": "^4.4.7",
     "@types/node": "^18.7.23",
+    "@types/semver": "^7.5.2",
     "jest": "^29.5.0",
     "jest-mock-extended": "^3.0.5",
     "ts-jest": "^29.1.0",

diff --git a/yarn-project/end-to-end/Dockerfile b/yarn-project/end-to-end/Dockerfile
@@ -1,22 +1,15 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder
 
-# Copy in the whole of yarn-project.
-COPY . .
-
 # Build aztec.js web.
 WORKDIR /usr/src/yarn-project/aztec.js
-RUN yarn build
-RUN yarn cache clean
+RUN yarn build:web
 
-# Build end-to-end.
 WORKDIR /usr/src/yarn-project/end-to-end
-RUN yarn build && yarn formatting
-
-# Prune dev dependencies. See comment in base image.
-RUN yarn cache clean
-RUN yarn workspaces focus --production > /dev/null
+# Productionify. See comment in yarn-project-base/Dockerfile.
+RUN yarn cache clean && yarn workspaces focus --production
 
 # Create final, minimal size image.
+# TODO: Not very minimal as chromium adds about 500MB of bloat :/ Separate or install at test runtime?
 FROM node:18-alpine
 RUN apk update && apk add --no-cache \
   bash \

diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json
@@ -40,6 +40,7 @@
     "@jest/globals": "^29.5.0",
     "@noble/curves": "^1.0.0",
     "@types/jest": "^29.5.0",
+    "@types/koa": "^2.13.9",
     "@types/koa-static": "^4.0.2",
     "@types/levelup": "^5.1.2",
     "@types/lodash.every": "^4.6.7",