feat: Add dependency resolver for noir_wasm and implement `FileMana…

…ger` for consistency with native interface (#3891) # Description Merging the work done here: AztecProtocol/aztec-packages#3696 #3781 #3760 Plus some extras to make the API nicer. ## Problem Closes(?) #3695 ## Summary Makes noir_wasm easier to work with, including dependency resolution and bundling. This package can be used from both node and the browser with identical API leveraging a virtual filesystem. Uses webpack for bundling, which is done in two steps: 1) rust -> wasm (cjs/esm) 2) TS + wasm (cjs/esm) -> universal package for web Tests have been migrated to mocha and playwright. ## Additional Context ~~I really want to test it [here](https://github.com/signorecello/noir-playground) before merging, but it's in a state in which it can be reviewed before we commit to an API.~~ Done: signorecello/noir-playground#32 Even though the initial memFS-backed FileManager developed by @alexghr is still here, it is not used for the web version due to import problems. The way it works now, webpack uses `memfs` directly it to alias the node `fs` module (which seems to be its intended use case) and allows us to use the nodejs `fs` API everywhere. ## Documentation Documentation is required for usage, but should basically be: ```typescript // Node.js import { compile, createFileManager } from '@noir-lang/noir_wasm'; // Rename!! const fm = createFileManager(myProjectPath); const myCompiledCode = await compile(fm); ``` ```typescript // Browser import { compile, createFileManager } from '@noir-lang/noir_wasm'; // Rename!! const fm = createFileManager('/'); for (const path of files) { await fm.writeFile(path, await getFileAsStream(path)); } const myCompiledCode = await compile(fm); ``` Check one: - [ ] No documentation needed. - [ ] Documentation included in this PR. - [ ] **[Exceptional Case]** Documentation to be submitted in a separate PR. # PR Checklist - [x] I have tested the changes locally. - [x] I have formatted the changes with [Prettier](https://prettier.io/) and/or `cargo fmt` on default settings. --------- Co-authored-by: sirasistant <sirasistant@gmail.com> Co-authored-by: Tom French <15848336+TomAFrench@users.noreply.github.com> Co-authored-by: Tom French <tom@tomfren.ch>
noir-lang · Jan 11, 2024 · c29c7d7 · c29c7d7
1 parent 5ae9d2f
commit c29c7d7
Show file tree

Hide file tree

Showing 66 changed files with 4,080 additions and 431 deletions.
diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml
@@ -47,35 +47,42 @@ jobs:
           retention-days: 3
 
   build-noir-wasm:
+    needs: [build-noirc-abi]
     runs-on: ubuntu-latest
     timeout-minutes: 30
 
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
 
-      - name: Setup Nix
-        uses: ./.github/actions/nix
+      - name: Setup toolchain
+        uses: dtolnay/rust-toolchain@1.71.1
+
+      - uses: Swatinem/rust-cache@v2
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          nix-cache-name: "noir"
-          cachix-auth-token: ${{ secrets.CACHIXAUTHTOKEN }}
+          key: noir-wasm
+          cache-on-failure: true
+          save-if: ${{ github.event_name != 'merge_group' }}
 
-      - name: Build wasm package
-        run: |
-          nix build -L .#noir_wasm
+      - name: Download noirc_abi_wasm package artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: noirc_abi_wasm
+          path: ./tooling/noirc_abi_wasm
 
-      - name: Dereference symlink
-        run: echo "UPLOAD_PATH=$(readlink -f ./result/noir_wasm)" >> $GITHUB_ENV
+      - name: Install Yarn dependencies
+        uses: ./.github/actions/setup
+
+      - name: Build noir_wasm
+        run: yarn workspace @noir-lang/noir_wasm build
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
           name: noir_wasm
-          path: ${{ env.UPLOAD_PATH }}
+          path: ./compiler/wasm
           retention-days: 3
 
-
   build-acvm-js:
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -289,25 +296,24 @@ jobs:
           name: noir_wasm
           path: ./compiler/wasm
 
+      - name: Install Yarn dependencies
+        uses: ./.github/actions/setup
+
       - name: Download nargo binary
         uses: actions/download-artifact@v3
         with:
           name: nargo
           path: ./nargo
 
-      - name: Compile fixtures with Nargo CLI
-        working-directory: ./compiler/wasm/fixtures
+      - name: Set nargo on PATH
         run: |
-          nargo_binary=${{ github.workspace }}/nargo/nargo
+          nargo_binary="${{ github.workspace }}/nargo/nargo"
           chmod +x $nargo_binary
-          for dir in $(ls -d */); do
-            pushd $dir/noir-script
-            $nargo_binary compile
-            popd
-          done
+          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
+          export PATH="$PATH:$(dirname $nargo_binary)"
 
-      - name: Install Yarn dependencies
-        uses: ./.github/actions/setup
+      - name: Build fixtures
+        run: yarn workspace @noir-lang/noir_wasm test:build_fixtures
 
       - name: Install Playwright
         uses: ./.github/actions/install-playwright

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -127,6 +127,7 @@ jsonrpc = { version = "0.16.0", features = ["minreq_http"] }
 tracing = "0.1.40"
 tracing-web = "0.1.3"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+rust-embed = "6.6.0"
 
 [profile.dev]
 # This is required to be able to run `cargo test` in acvm_js due to the `locals exceeds maximum` error.

diff --git a/compiler/integration-tests/test/browser/compile_prove_verify.test.ts b/compiler/integration-tests/test/browser/compile_prove_verify.test.ts
@@ -1,22 +1,13 @@
 import { expect } from '@esm-bundle/chai';
 import * as TOML from 'smol-toml';
 
-import newCompiler, {
-  CompiledProgram,
-  PathToFileSourceMap,
-  compile,
-  init_log_level as compilerLogLevel,
-} from '@noir-lang/noir_wasm';
+import { compile, createFileManager } from '@noir-lang/noir_wasm';
 import { Noir } from '@noir-lang/noir_js';
 import { InputMap } from '@noir-lang/noirc_abi';
 import { BarretenbergBackend } from '@noir-lang/backend_barretenberg';
 
 import { getFile } from './utils.js';
 
-await newCompiler();
-
-compilerLogLevel('INFO');
-
 const test_cases = [
   {
     case: 'test_programs/execution_success/1_mul',
@@ -32,12 +23,11 @@ const suite = Mocha.Suite.create(mocha.suite, 'Noir end to end test');
 
 suite.timeout(60 * 20e3); //20mins
 
-function getCircuit(noirSource: string): CompiledProgram {
-  const sourceMap = new PathToFileSourceMap();
-  sourceMap.add_source_code('main.nr', noirSource);
-
-  // We're ignoring this in the resolver but pass in something sensible.
-  const result = compile('main.nr', undefined, undefined, sourceMap);
+async function getCircuit(projectPath: string) {
+  const fm = createFileManager('/');
+  await fm.writeFile('./src/main.nr', await getFile(`${projectPath}/src/main.nr`));
+  await fm.writeFile('./Nargo.toml', await getFile(`${projectPath}/Nargo.toml`));
+  const result = await compile(fm);
   if (!('program' in result)) {
     throw new Error('Compilation failed');
   }
@@ -51,11 +41,9 @@ test_cases.forEach((testInfo) => {
     const base_relative_path = '../../../../..';
     const test_case = testInfo.case;
 
-    const noir_source = await getFile(`${base_relative_path}/${test_case}/src/main.nr`);
-
-    let noir_program: CompiledProgram;
+    let noir_program;
     try {
-      noir_program = getCircuit(noir_source);
+      noir_program = await getCircuit(`${base_relative_path}/${test_case}`);
 
       expect(noir_program, 'Compile output ').to.be.an('object');
     } catch (e) {
@@ -66,7 +54,7 @@ test_cases.forEach((testInfo) => {
     const backend = new BarretenbergBackend(noir_program);
     const program = new Noir(noir_program, backend);
 
-    const prover_toml = await getFile(`${base_relative_path}/${test_case}/Prover.toml`);
+    const prover_toml = await new Response(await getFile(`${base_relative_path}/${test_case}/Prover.toml`)).text();
     const inputs: InputMap = TOML.parse(prover_toml) as InputMap;
 
     // JS Proving

diff --git a/compiler/integration-tests/test/browser/recursion.test.ts b/compiler/integration-tests/test/browser/recursion.test.ts
@@ -2,59 +2,46 @@
 import { expect } from '@esm-bundle/chai';
 import { TEST_LOG_LEVEL } from '../environment.js';
 import { Logger } from 'tslog';
-import newCompiler, {
-  CompiledProgram,
-  PathToFileSourceMap,
-  compile,
-  init_log_level as compilerLogLevel,
-} from '@noir-lang/noir_wasm';
 import { acvm, abi, Noir } from '@noir-lang/noir_js';
 
 import * as TOML from 'smol-toml';
 import { BarretenbergBackend } from '@noir-lang/backend_barretenberg';
 import { getFile } from './utils.js';
 import { Field, InputMap } from '@noir-lang/noirc_abi';
+import { createFileManager, compile } from '@noir-lang/noir_wasm';
 
 const logger = new Logger({ name: 'test', minLevel: TEST_LOG_LEVEL });
 
 const { default: initACVM } = acvm;
 const { default: newABICoder } = abi;
 
-await newCompiler();
 await newABICoder();
 await initACVM();
 
-compilerLogLevel('INFO');
-
 const base_relative_path = '../../../../..';
 const circuit_main = 'test_programs/execution_success/assert_statement';
 const circuit_recursion = 'compiler/integration-tests/circuits/recursion';
 
-function getCircuit(noirSource: string): CompiledProgram {
-  const sourceMap = new PathToFileSourceMap();
-  sourceMap.add_source_code('main.nr', noirSource);
-  const result = compile('main.nr', undefined, undefined, sourceMap);
+async function getCircuit(projectPath: string) {
+  const fm = createFileManager('/');
+  await fm.writeFile('./src/main.nr', await getFile(`${projectPath}/src/main.nr`));
+  await fm.writeFile('./Nargo.toml', await getFile(`${projectPath}/Nargo.toml`));
+  const result = await compile(fm);
   if (!('program' in result)) {
     throw new Error('Compilation failed');
   }
-
   return result.program;
 }
 
 describe('It compiles noir program code, receiving circuit bytes and abi object.', () => {
-  let circuit_main_source;
   let circuit_main_toml;
-  let circuit_recursion_source;
 
   before(async () => {
-    circuit_main_source = await getFile(`${base_relative_path}/${circuit_main}/src/main.nr`);
-    circuit_main_toml = await getFile(`${base_relative_path}/${circuit_main}/Prover.toml`);
-
-    circuit_recursion_source = await getFile(`${base_relative_path}/${circuit_recursion}/src/main.nr`);
+    circuit_main_toml = await new Response(await getFile(`${base_relative_path}/${circuit_main}/Prover.toml`)).text();
   });
 
   it('Should generate valid inner proof for correct input, then verify proof within a proof', async () => {
-    const main_program = getCircuit(circuit_main_source);
+    const main_program = await getCircuit(`${base_relative_path}/${circuit_main}`);
     const main_inputs: InputMap = TOML.parse(circuit_main_toml) as InputMap;
 
     const main_backend = new BarretenbergBackend(main_program);
@@ -83,7 +70,7 @@ describe('It compiles noir program code, receiving circuit bytes and abi object.
 
     logger.debug('recursion_inputs', recursion_inputs);
 
-    const recursion_program = await getCircuit(circuit_recursion_source);
+    const recursion_program = await getCircuit(`${base_relative_path}/${circuit_recursion}`);
 
     const recursion_backend = new BarretenbergBackend(recursion_program);
 

diff --git a/compiler/integration-tests/test/browser/utils.ts b/compiler/integration-tests/test/browser/utils.ts
@@ -1,8 +1,8 @@
-export async function getFile(file_path: string): Promise<string> {
+export async function getFile(file_path: string): Promise<ReadableStream<Uint8Array>> {
   const file_url = new URL(file_path, import.meta.url);
   const response = await fetch(file_url);
 
   if (!response.ok) throw new Error('Network response was not OK');
 
-  return await response.text();
+  return response.body as ReadableStream<Uint8Array>;
 }
diff --git a/compiler/integration-tests/test/node/onchain_recursive_verification.test.ts b/compiler/integration-tests/test/node/onchain_recursive_verification.test.ts
@@ -2,44 +2,45 @@ import { expect } from 'chai';
 import { ethers } from 'hardhat';
 
 import { readFileSync } from 'node:fs';
-import { resolve } from 'path';
+import { resolve, join } from 'path';
 import toml from 'toml';
 
-import {
-  compile,
-  CompiledProgram,
-  init_log_level as compilerLogLevel,
-  PathToFileSourceMap,
-} from '@noir-lang/noir_wasm';
 import { Noir } from '@noir-lang/noir_js';
 import { BarretenbergBackend } from '@noir-lang/backend_barretenberg';
 import { Field, InputMap } from '@noir-lang/noirc_abi';
 
-compilerLogLevel('INFO');
+import { compile, createFileManager } from '@noir-lang/noir_wasm';
 
 it(`smart contract can verify a recursive proof`, async () => {
-  const innerSourcePath = resolve(`../../test_programs/execution_success/assert_statement/src/main.nr`);
-  const sourceMapInnerProgram = new PathToFileSourceMap();
-  sourceMapInnerProgram.add_source_code(innerSourcePath, readFileSync(innerSourcePath, 'utf-8'));
-  const innerProgram = (
-    compile(innerSourcePath, undefined, undefined, sourceMapInnerProgram) as { program: CompiledProgram }
-  ).program;
-
-  const recursionSourcePath = resolve(`./circuits/recursion/src/main.nr`);
-  const sourceMapRecursionProgram = new PathToFileSourceMap();
-  sourceMapRecursionProgram.add_source_code(recursionSourcePath, readFileSync(recursionSourcePath, 'utf-8'));
-  const recursionProgram = (
-    compile(recursionSourcePath, undefined, undefined, sourceMapRecursionProgram) as { program: CompiledProgram }
-  ).program;
+  const basePath = resolve(join(__dirname, '../../../../'));
+  const fm = createFileManager(basePath);
+  const innerCompilationResult = await compile(
+    fm,
+    join(basePath, './test_programs/execution_success/assert_statement'),
+  );
+  if (!('program' in innerCompilationResult)) {
+    throw new Error('Compilation failed');
+  }
+  const innerProgram = innerCompilationResult.program;
+
+  const recursionCompilationResult = await compile(
+    fm,
+    join(basePath, './compiler/integration-tests/circuits/recursion'),
+  );
+  if (!('program' in recursionCompilationResult)) {
+    throw new Error('Compilation failed');
+  }
+  const recursionProgram = recursionCompilationResult.program;
 
   // Intermediate proof
 
   const inner_backend = new BarretenbergBackend(innerProgram);
   const inner = new Noir(innerProgram);
 
   const inner_prover_toml = readFileSync(
-    resolve(`../../test_programs/execution_success/assert_statement/Prover.toml`),
+    join(basePath, `./test_programs/execution_success/assert_statement/Prover.toml`),
   ).toString();
+
   const inner_inputs = toml.parse(inner_prover_toml);
 
   const { witness: main_witness } = await inner.execute(inner_inputs);

diff --git a/compiler/integration-tests/test/node/smart_contract_verifier.test.ts b/compiler/integration-tests/test/node/smart_contract_verifier.test.ts
@@ -5,11 +5,10 @@ import { readFileSync } from 'node:fs';
 import { resolve } from 'path';
 import toml from 'toml';
 
-import { PathToFileSourceMap, compile, init_log_level as compilerLogLevel } from '@noir-lang/noir_wasm';
 import { Noir } from '@noir-lang/noir_js';
 import { BarretenbergBackend } from '@noir-lang/backend_barretenberg';
 
-compilerLogLevel('INFO');
+import { compile, createFileManager } from '@noir-lang/noir_wasm';
 
 const test_cases = [
   {
@@ -31,11 +30,8 @@ test_cases.forEach((testInfo) => {
     const base_relative_path = '../..';
     const test_case = testInfo.case;
 
-    const noirSourcePath = resolve(`${base_relative_path}/${test_case}/src/main.nr`);
-    const sourceMap = new PathToFileSourceMap();
-    sourceMap.add_source_code(noirSourcePath, readFileSync(noirSourcePath, 'utf-8'));
-
-    const compileResult = compile(noirSourcePath, undefined, undefined, sourceMap);
+    const fm = createFileManager(resolve(`${base_relative_path}/${test_case}`));
+    const compileResult = await compile(fm);
     if (!('program' in compileResult)) {
       throw new Error('Compilation failed');
     }

diff --git a/compiler/noirc_driver/Cargo.toml b/compiler/noirc_driver/Cargo.toml
@@ -21,7 +21,7 @@ iter-extended.workspace = true
 fm.workspace = true
 serde.workspace = true
 fxhash.workspace = true
-rust-embed = "6.6.0"
+rust-embed.workspace = true
 tracing.workspace = true
 
 aztec_macros = { path = "../../aztec_macros" }
diff --git a/compiler/wasm/.eslintrc.js b/compiler/wasm/.eslintrc.js
@@ -1,3 +1,3 @@
 module.exports = {
-  extends: ["../../.eslintrc.js"],
+  extends: ['../../.eslintrc.js'],
 };
diff --git a/compiler/wasm/.gitignore b/compiler/wasm/.gitignore
@@ -1 +1,3 @@
 noir-script/target
+dist
+build
diff --git a/compiler/wasm/.mocharc.json b/compiler/wasm/.mocharc.json
@@ -1,7 +1,13 @@
 {
-  "extension": [
-    "ts"
-  ],
-  "spec": "test/node/**/*.test.ts",
-  "require": "ts-node/register"
-}
+    "require": "ts-node/register",
+    "extensions": [
+      "ts"
+    ],
+    "spec": [
+      "./test/**/!(browser)/*.test.ts"
+    ],
+    "node-option": [
+      "loader=ts-node"
+    ]
+  }
+