Merge branch 'master' into alexg/feat/snapshot-trees

.circleci/config.yml

@@ -676,6 +676,17 @@ jobs:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_escrow_contract.test.ts
 
+  e2e-inclusion-proofs-contract:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Test"
+          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_inclusion_proofs_contract.test.ts
+
   e2e-pending-commitments-contract:
     docker:
       - image: aztecprotocol/alpine-build-image
@@ -1113,6 +1124,7 @@ workflows:
       - e2e-public-to-private-messaging: *e2e_test
       - e2e-account-contracts: *e2e_test
       - e2e-escrow-contract: *e2e_test
+      - e2e-inclusion-proofs-contract: *e2e_test
       - e2e-pending-commitments-contract: *e2e_test
       - e2e-ordering: *e2e_test
       - uniswap-trade-on-l1-from-l2: *e2e_test
@@ -1148,6 +1160,7 @@ workflows:
             - e2e-public-to-private-messaging
             - e2e-account-contracts
             - e2e-escrow-contract
+            - e2e-inclusion-proofs-contract
             - e2e-pending-commitments-contract
             - e2e-ordering
             - uniswap-trade-on-l1-from-l2

barretenberg/acir_tests/Dockerfile.bb

@@ -4,7 +4,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-acir-tests as noir-acir-t
 FROM node:18-alpine
 RUN apk update && apk add git bash curl jq coreutils
 COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build
-COPY --from=noir-acir-tests /usr/src/noir/tooling/nargo_cli/tests /usr/src/noir/tooling/nargo_cli/tests
+COPY --from=noir-acir-tests /usr/src/noir/test_programs /usr/src/noir/test_programs
 WORKDIR /usr/src/barretenberg/acir_tests
 COPY . .
 # Run every acir test through native bb build prove_then_verify flow.

barretenberg/acir_tests/Dockerfile.bb.js

@@ -3,7 +3,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-acir-tests as noir-acir-t
 
 FROM node:18
 COPY --from=0 /usr/src/barretenberg/ts /usr/src/barretenberg/ts
-COPY --from=noir-acir-tests /usr/src/noir/tooling/nargo_cli/tests /usr/src/noir/tooling/nargo_cli/tests
+COPY --from=noir-acir-tests /usr/src/noir/test_programs /usr/src/noir/test_programs
 RUN apt update && apt install -y lsof jq
 WORKDIR /usr/src/barretenberg/acir_tests
 # Build/install ts apps.

barretenberg/acir_tests/Dockerfile.bb.sol

@@ -6,7 +6,7 @@ FROM node:18-alpine
 RUN apk update && apk add git bash curl jq
 COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build
 COPY --from=1 /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol
-COPY --from=noir-acir-tests /usr/src/noir/tooling/nargo_cli/tests /usr/src/noir/tooling/nargo_cli/tests
+COPY --from=noir-acir-tests /usr/src/noir/test_programs /usr/src/noir/test_programs
 COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/anvil /usr/local/bin/anvil
 WORKDIR /usr/src/barretenberg/acir_tests
 COPY . .

barretenberg/acir_tests/Dockerfile.noir_acir_tests

@@ -5,6 +5,6 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir
 RUN apk add bash jq
 ENV PATH="/usr/src/noir/target/release:${PATH}"
-WORKDIR /usr/src/noir/tooling/nargo_cli/tests
+WORKDIR /usr/src/noir/test_programs
 COPY . .
 RUN ./rebuild.sh

barretenberg/acir_tests/clone_test_vectors.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -eu
 
-TEST_SRC=${TEST_SRC:-../../noir/tooling/nargo_cli/tests/acir_artifacts}
+TEST_SRC=${TEST_SRC:-../../noir/test_programs/acir_artifacts}
 
 if [ ! -d acir_tests ]; then
   cp -R $TEST_SRC acir_tests

barretenberg/cpp/.dockerignore

@@ -17,5 +17,6 @@
 
 # Needed scripts.
 !scripts/install-wasi-sdk.sh
+!scripts/strip-wasm.sh
 !./.clang-format
 !./format.sh

barretenberg/cpp/dockerfiles/Dockerfile.wasm-linux-clang

@@ -6,15 +6,10 @@ RUN ./scripts/install-wasi-sdk.sh
 COPY . .
 RUN cmake --preset wasm && cmake --build --preset wasm
 RUN cmake --preset wasm-threads && cmake --build --preset wasm-threads
+RUN ./scripts/strip-wasm.sh
 
 FROM scratch
 WORKDIR /usr/src/barretenberg/cpp
 COPY . .
 COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build-wasm/bin/barretenberg.wasm
 COPY --from=builder /usr/src/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm
-# Copy libs for consuming projects.
-COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/lib/libbarretenberg.a /usr/src/barretenberg/cpp/build-wasm/lib/libbarretenberg.a
-COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/lib/libwasi.a /usr/src/barretenberg/cpp/build-wasm/lib/libwasi.a
-COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/lib/libenv.a /usr/src/barretenberg/cpp/build-wasm/lib/libenv.a
-# Copy wasi-sdk so that consuming projects have the toolchain available.
-COPY --from=builder /usr/src/barretenberg/cpp/src/wasi-sdk-20.0 /usr/src/barretenberg/cpp/src/wasi-sdk-20.0

barretenberg/ts/.gitignore

@@ -5,3 +5,5 @@ dest
 .tsbuildinfo*
 *.log
 /crs
+package.tgz
+package

barretenberg/ts/Dockerfile

@@ -2,6 +2,9 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-wasm-linux-clang
 
 FROM node:18-alpine
 COPY --from=0 /usr/src/barretenberg /usr/src/barretenberg
+
+# Create a standalone container that can run bb.js (and tests).
+# We'll perform the build in a new, different directory, so the original directory can become the "published" package.
 WORKDIR /usr/src/barretenberg/ts
 # Leverage layer caching. Only re-install packages if these files change.
 COPY .yarn .yarn
@@ -12,3 +15,6 @@ RUN yarn --immutable
 COPY . .
 RUN yarn formatting && SKIP_CPP_BUILD=1 yarn build
 CMD ["yarn", "test"]
+
+# We want to create a pure package, as would be published to npm, for consuming projects.
+RUN yarn pack && tar zxf package.tgz && rm package.tgz

barretenberg/ts/package.json

@@ -20,19 +20,19 @@
     "README.md"
   ],
   "scripts": {
-    "clean": "rm -rf ./dest .tsbuildinfo .tsbuildinfo.cjs",
-    "build": "yarn clean && yarn build:wasm && yarn build:esm && yarn build:cjs && yarn build:browser",
+    "clean": "rm -rf ./dest .tsbuildinfo .tsbuildinfo.cjs package.tgz package",
+    "build": "yarn clean && yarn build:wasm && yarn build:esm && yarn build:cjs && yarn build:browser && yarn build:package",
     "build:wasm": "./scripts/build_wasm.sh",
     "build:esm": "tsc -b && chmod +x ./dest/node/main.js",
     "build:cjs": "tsc -b tsconfig.cjs.json && ./scripts/cjs_postprocess.sh",
     "build:browser": "webpack",
     "build:bindings": "cd .. && ./scripts/bindgen.sh",
+    "build:package": "yarn pack && tar zxf package.tgz && rm -f package.tgz",
     "formatting": "prettier --check ./src && eslint --max-warnings 0 ./src",
     "formatting:fix": "prettier -w ./src",
     "test": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests",
     "test:debug": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 node --inspect-brk=0.0.0.0 --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests --runInBand",
     "simple_test": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 node ./src/examples/simple.rawtest.ts",
-    "prepack": "yarn build",
     "deploy": "npm publish --access public"
   },
   "jest": {

barretenberg/ts/src/barretenberg/index.ts

@@ -3,7 +3,7 @@ import { BarretenbergApi, BarretenbergApiSync } from '../barretenberg_api/index.
 import { createMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/factory/node/index.js';
 import { BarretenbergWasmMain, BarretenbergWasmMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/index.js';
 import { getRemoteBarretenbergWasm } from '../barretenberg_wasm/helpers/index.js';
-import { BarretenbergWasmWorker } from '../barretenberg_wasm/index.js';
+import { BarretenbergWasmWorker, fetchModuleAndThreads } from '../barretenberg_wasm/index.js';
 import createDebug from 'debug';
 
 const debug = createDebug('bb.js:wasm');
@@ -19,14 +19,15 @@ export class Barretenberg extends BarretenbergApi {
 
   /**
    * Constructs an instance of Barretenberg.
-   * Launches it within a worker. This is necessary as it block waiting on child threads to complete,
+   * Launches it within a worker. This is necessary as it blocks waiting on child threads to complete,
    * and blocking the main thread in the browser is not allowed.
    * It threads > 1 (defaults to hardware availability), child threads will be created on their own workers.
    */
-  static async new(threads?: number) {
+  static async new(desiredThreads?: number) {
     const worker = createMainWorker();
     const wasm = getRemoteBarretenbergWasm<BarretenbergWasmMainWorker>(worker);
-    await wasm.init(threads, proxy(debug));
+    const { module, threads } = await fetchModuleAndThreads(desiredThreads);
+    await wasm.init(module, threads, proxy(debug));
     return new Barretenberg(worker, wasm);
   }
 
@@ -49,7 +50,8 @@ export class BarretenbergSync extends BarretenbergApiSync {
 
   static async new() {
     const wasm = new BarretenbergWasmMain();
-    await wasm.init(1);
+    const { module, threads } = await fetchModuleAndThreads(1);
+    await wasm.init(module, threads);
     return new BarretenbergSync(wasm);
   }
 

barretenberg/ts/src/barretenberg_wasm/barretenberg_wasm_main/index.ts

@@ -2,7 +2,6 @@ import { type Worker } from 'worker_threads';
 import createDebug from 'debug';
 import { Remote } from 'comlink';
 import { getNumCpu, getRemoteBarretenbergWasm, getSharedMemoryAvailable } from '../helpers/index.js';
-import { fetchCode } from '../fetch_code/index.js';
 import { createThreadWorker } from '../barretenberg_wasm_thread/factory/node/index.js';
 import { type BarretenbergWasmThreadWorker } from '../barretenberg_wasm_thread/index.js';
 import { BarretenbergWasmBase } from '../barretenberg_wasm_base/index.js';
@@ -30,6 +29,7 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
    * Init as main thread. Spawn child threads.
    */
   public async init(
+    module: WebAssembly.Module,
     threads = Math.min(getNumCpu(), BarretenbergWasmMain.MAX_THREADS),
     logger: (msg: string) => void = debug,
     initial = 25,
@@ -41,10 +41,6 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
     const maxMb = (maximum * 2 ** 16) / (1024 * 1024);
     const shared = getSharedMemoryAvailable();
 
-    if (!shared) {
-      threads = 1;
-    }
-
     this.logger(
       `initial mem: ${initial} pages, ${initialMb}MiB. ` +
         `max mem: ${maximum} pages, ${maxMb}MiB. ` +
@@ -53,8 +49,7 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
 
     this.memory = new WebAssembly.Memory({ initial, maximum, shared });
 
-    const code = await fetchCode(shared);
-    const { instance, module } = await WebAssembly.instantiate(code, this.getImportObj(this.memory));
+    const instance = await WebAssembly.instantiate(module, this.getImportObj(this.memory));
 
     this.instance = instance;
 

barretenberg/ts/src/barretenberg_wasm/helpers/node/index.ts

@@ -17,8 +17,11 @@ export function getRemoteBarretenbergWasm<T>(worker: Worker): T {
   return wrap(nodeEndpoint(worker)) as T;
 }
 
+/**
+ * Returns number of cpus as reported by the system, unless overriden by HARDWARE_CONCURRENCY env var.
+ */
 export function getNumCpu() {
-  return os.cpus().length;
+  return +process.env.HARDWARE_CONCURRENCY! || os.cpus().length;
 }
 
 /**

barretenberg/ts/src/barretenberg_wasm/index.ts

@@ -1,20 +1,30 @@
 import { proxy } from 'comlink';
 import createDebug from 'debug';
 import { createMainWorker } from './barretenberg_wasm_main/factory/node/index.js';
-import { getRemoteBarretenbergWasm } from './helpers/node/index.js';
+import { getRemoteBarretenbergWasm, getSharedMemoryAvailable } from './helpers/node/index.js';
 import { BarretenbergWasmMain, BarretenbergWasmMainWorker } from './barretenberg_wasm_main/index.js';
+import { fetchCode } from './fetch_code/index.js';
 
 const debug = createDebug('bb.js:wasm');
 
+export async function fetchModuleAndThreads(desiredThreads?: number) {
+  const shared = getSharedMemoryAvailable();
+  const threads = shared ? desiredThreads : 1;
+  const code = await fetchCode(shared);
+  const module = await WebAssembly.compile(code);
+  return { module, threads };
+}
+
 export class BarretenbergWasm extends BarretenbergWasmMain {
   /**
    * Construct and initialize BarretenbergWasm within a Worker. Return both the worker and the wasm proxy.
    * Used when running in the browser, because we can't block the main thread.
    */
-  public static async new(threads?: number) {
+  public static async new(desiredThreads?: number) {
     const worker = createMainWorker();
     const wasm = getRemoteBarretenbergWasm<BarretenbergWasmMainWorker>(worker);
-    await wasm.init(threads, proxy(debug));
+    const { module, threads } = await fetchModuleAndThreads(desiredThreads);
+    await wasm.init(module, threads, proxy(debug));
     return { worker, wasm };
   }
 }