fix(bb.js): (breaking change) bundles bb.js properly so that it works…

… in the browser and in node (#1855)

* Reverts prior work and builds upon original webpack.
* Bundles works and wasms into the output bundle. ~12MB but compresses
with e.g. brotli to about 1.5MB (consumers responsbility). This required
a fair bit of refactoring of the bb_wasm ts code into `base`, `main` and
`thread` to avoid what webpack saw as circular dependency.
* Manually testing and working with consumers, zero config webpack,
vite, next.js, hardhat.
* Automatically tests:
  * bb: all acir test vectors through prove_and_verify
  * bb and bb.js: all command line interface commands.
* bb.js: a single test through single and multithreaded on chrome and
webkit.
* bootstrap_docker.sh can be used to build and run these tests as per
CCI.

---------

Co-authored-by: ludamad <adam.domurad@gmail.com>

.circleci/config.yml

@@ -240,6 +240,17 @@ jobs:
           name: "Test"
           command: ./scripts/ci/store_test_benchmark_logs $AZTEC_GITHUB_TOKEN
 
+  barretenberg-acir-tests-bb:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Build and test"
+          command: cond_spot_run_build barretenberg-acir-tests-bb 32
+
   bb-js:
     machine:
       image: ubuntu-2004:202010-01
@@ -248,7 +259,7 @@ jobs:
       - *checkout
       - *setup_env
       - run:
-          name: "Build"
+          name: "Build and test"
           command: build bb.js
 
   bb-js-tests:
@@ -259,10 +270,10 @@ jobs:
       - *checkout
       - *setup_env
       - run:
-          name: "Test"
+          name: "Build and test"
           command: cond_spot_run_tests bb.js
 
-  barretenberg-acir-tests:
+  bb-js-acir-tests:
     docker:
       - image: aztecprotocol/alpine-build-image
     resource_class: small
@@ -271,7 +282,7 @@ jobs:
       - *setup_env
       - run:
           name: "Build and test"
-          command: cond_spot_run_build barretenberg-acir-tests 32
+          command: cond_spot_run_build barretenberg-acir-tests-bb.js 32
 
   circuits-wasm-linux-clang:
     docker:
@@ -1279,9 +1290,12 @@ workflows:
           requires:
             - bb-js
           <<: *defaults
-      - barretenberg-acir-tests:
+      - barretenberg-acir-tests-bb:
           requires:
             - barretenberg-x86_64-linux-clang-assert
+          <<: *defaults
+      - bb-js-acir-tests:
+          requires:
             - bb-js
           <<: *defaults
       - circuits-wasm-linux-clang: *defaults

.vscode/settings.json

@@ -113,7 +113,9 @@
     "**/.yarn": true,
     "**/.yalc": true,
     "**/node_modules": true,
-    "**/.pnp.*": true
+    "**/.pnp.*": true,
+    "circuits/cpp/barretenberg/**": true,
+    "**/msgpack-c/**": true
   },
   "[terraform]": {
     "editor.defaultFormatter": "hashicorp.terraform"
@@ -128,8 +130,8 @@
   // C++/Circuits settings
   ///////////////////////////////////////
   "[cpp]": {
-     "editor.defaultFormatter": "llvm-vs-code-extensions.vscode-clangd",
-      "editor.formatOnSave": true,
+    "editor.defaultFormatter": "llvm-vs-code-extensions.vscode-clangd",
+    "editor.formatOnSave": true
   },
   //
   // Clangd. Note that this setting may be overridden by user settings
@@ -164,7 +166,7 @@
   // which ensures SRS can be read
   "testMate.cpp.test.workingDirectory": "${workspaceFolder}/circuits/cpp/build",
   // Filter all binaries that are not tests
-  "testMate.cpp.test.executables": "${workspaceFolder}/circuits/cpp/build/bin/*{test,Test,TEST}*",
+  "testMate.cpp.test.executables": "${workspaceFolder}/circuits/cpp/build/bin/*{test,Test,TEST}*"
   // End C++/Circuits settings
   ///////////////////////////////////////
 }

barretenberg

@@ -0,0 +1 @@
+./circuits/cpp/barretenberg

bootstrap_docker.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
-# This script builds the projects listed in build_mainifest.sh, terminating when it reaches TARGET_PROJECT.
+# This script builds the projects listed in build_mainifest.sh, terminating when it reaches PROJECT_NAME.
 # If run from within a project, it will build only that project, unless env var ONLY_TARGET=false.
 #
+# Usage:
+#   boostrap_docker.sh [PROJECT_NAME]
+#
 # To build everything in build_manifest.sh:
 #   bootstrap_docker.sh
 #
@@ -18,24 +21,25 @@
 
 set -e
 
-TARGET_PROJECT=$1
+PROJECT_NAME=$1
 COMMIT_HASH=$(git rev-parse HEAD)
+ONLY_TARGET=${ONLY_TARGET:-}
 
 # If we're calling this script from within a project directory, that's the target project.
-if [ -z "$TARGET_PROJECT" ]; then
+if [ -z "$PROJECT_NAME" ]; then
   PATH_PREFIX=$(git rev-parse --show-prefix)
   if [ -n "$PATH_PREFIX" ]; then
     # We are in a project folder.
     ONLY_TARGET=${ONLY_TARGET:-true}
-    TARGET_PROJECT=$(basename $PATH_PREFIX)
+    PROJECT_NAME=$(basename $PATH_PREFIX)
     cd $(git rev-parse --show-cdup)
   fi
 fi
 
-source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER 
-build_local $TARGET_PROJECT $ONLY_TARGET
+source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER
+build_local $PROJECT_NAME $ONLY_TARGET
 
-if [ -z "$TARGET_PROJECT" ]; then
+if [ -z "$PROJECT_NAME" ]; then
   echo
   echo "Success! You could now run e.g.:"
   echo "  docker run -ti --rm aztecprotocol/end-to-end:latest e2e_private_token_contract.test"

build-system/scripts/build_local

@@ -9,7 +9,11 @@
 set -eu
 
 TARGET_PROJECT=$1
-ONLY_TARGET=$2
+ONLY_TARGET=${2:-}
+
+if [ -n "${NO_CACHE:-}" ]; then
+  ADDITIONAL_ARGS=--no-cache
+fi
 
 ROOT=$(git rev-parse --show-toplevel)
 source $ROOT/build_manifest.sh
@@ -20,6 +24,7 @@ for E in ${PROJECTS[@]}; do
   WORKING_DIR=${ARR[1]}
   DOCKERFILE=${ARR[2]}
   REPO=${ARR[3]:-$PROJECT_DIR_NAME}
+  LAUNCH=${ARR[4]:-}
 
   if [ "$ONLY_TARGET" == "true" -a ! "$PROJECT_DIR_NAME" = "$TARGET_PROJECT" ]; then
     continue
@@ -44,9 +49,13 @@ for E in ${PROJECTS[@]}; do
   echo "*** Building $PROJECT_DIR_NAME:$DOCKERFILE -> $REPO ***"
   echo
 
-  time docker build --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $ECR_DEPLOY_URL/$REPO:latest .
+  time docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $ECR_DEPLOY_URL/$REPO:latest .
   docker tag $ECR_DEPLOY_URL/$REPO:latest aztecprotocol/$REPO:latest
 
+  if [ -n "$LAUNCH" ]; then
+    docker run -ti --rm aztecprotocol/$REPO:latest
+  fi
+
   if [ "$PROJECT_DIR_NAME" = "$TARGET_PROJECT" ]; then
     break
   fi

build-system/scripts/setup_env

@@ -14,6 +14,7 @@ JOB_NAME=$3
 GIT_REPOSITORY_URL=${4:-}
 BRANCH=${5:-}
 
+BASH_ENV=${BASH_ENV:-}
 BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 PROJECT=$(cat PROJECT)
 VERSION=$(cat VERSION)
@@ -95,7 +96,7 @@ source $BASH_ENV
 if [ -n "$COMMIT_HASH" ]; then
   # Install and ensure correct permissions on build instance key.
   mkdir -p ~/.ssh
-  echo $BUILD_INSTANCE_KEY | base64 -d > ~/.ssh/build_instance_key
+  echo ${BUILD_INSTANCE_KEY:-} | base64 -d > ~/.ssh/build_instance_key
   chmod 600 ~/.ssh/build_instance_key
 fi