chore: Plumbs noir subrepo into yarn-project. (#3420)

For a less noisey review:
#3427
It contains everything prior to the noir pulls. This is PR we actually
want to merge though.


* Adds `noir-packages` job to build the ts packages.
* Fix `sed` use in bb.js to work on Mac.
* Bootstrap scripts for noir, call it from bootstrap.sh.
* Fix to `build_local`.
* Use the noir bootstrap scripts to build the docker containers.
* `yarn-project` now builds the protocol circuits from the noir build in
bootstrap and dockerfile.
* `yarn-project` portals the noir packages in. **What does this mean for
releases!?*
* Fixes as we're now on noir 19.
* Remove `noir-protocol-circuits/src/target` from repo as they're now
built.
* Bootstrap improvements:
  * Defaults to a "lighter" bootstrap.
  * `bootstrap.sh clean` resets repo to as if fresh clone. (be careful).
* Introduces ability to test the bootstrap process via a docker
container. (might add a nightly job, plus alternative ubuntu releases).

.circleci/config.yml

@@ -103,6 +103,17 @@ jobs:
           name: "Create ECR manifest"
           command: create_ecr_manifest noir x86_64,arm64
 
+  noir-packages:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Build"
+          command: cond_spot_run_build noir-packages 32
+
   # Barretenberg
   barretenberg-wasm-linux-clang:
     docker:
@@ -1013,6 +1024,7 @@ workflows:
             - noir-x86_64
             - noir-arm64
           <<: *defaults
+      - noir-packages: *defaults
 
       # Barretenberg
       - barretenberg-x86_64-linux-gcc: *defaults
@@ -1069,6 +1081,8 @@ workflows:
             - circuits-wasm-linux-clang
             - l1-contracts
             - bb-js
+            - noir-ecr-manifest
+            - noir-packages
           <<: *defaults
       - yarn-project:
           requires:
@@ -1147,7 +1161,6 @@ workflows:
       - end:
           requires:
             - mainnet-fork
-            - noir-ecr-manifest
             - e2e-2-pxes
             - e2e-deploy-contract
             - e2e-lending-contract

barretenberg/bootstrap.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
 set -eu
 
-# Navigate to script folder
 cd "$(dirname "$0")"
 
-(cd cpp && ./bootstrap.sh)
-(cd ts && yarn install --immutable && yarn build && npm link)
+(cd cpp && ./bootstrap.sh $@)
+(cd ts && ./bootstrap.sh $@)

barretenberg/cpp/bootstrap.sh

@@ -4,22 +4,32 @@ set -eu
 # Navigate to script folder
 cd "$(dirname "$0")"
 
-# Clean.
-rm -rf ./build
-rm -rf ./build-wasm
+CMD=${1:-}
+
+if [ -n "$CMD" ]; then
+  if [ "$CMD" = "clean" ]; then
+    git clean -ffdx
+    exit 0
+  else
+    echo "Unknown command: $CMD"
+    exit 1
+  fi
+fi
 
 # Determine system.
 if [[ "$OSTYPE" == "darwin"* ]]; then
   OS=macos
 elif [[ "$OSTYPE" == "linux-gnu" ]]; then
   OS=linux
+elif [[ "$OSTYPE" == "linux-musl" ]]; then
+  OS=linux
 else
   echo "Unknown OS: $OSTYPE"
   exit 1
 fi
 
 # Download ignition transcripts.
-(cd ./srs_db && ./download_ignition.sh 3)
+(cd ./srs_db && ./download_ignition.sh 0)
 
 # Pick native toolchain file.
 ARCH=$(uname -m)
@@ -33,22 +43,23 @@ else
   fi
 fi
 
+# Remove cmake cache files.
+rm -f {build,build-wasm,build-wasm-threads}/CMakeCache.txt
+
 echo "#################################"
 echo "# Building with preset: $PRESET"
 echo "# When running cmake directly, remember to use: --build --preset $PRESET"
 echo "#################################"
 
 # Build native.
 cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert
-cmake --build --preset $PRESET ${@/#/--target }
-
-cd ./build
-# The Grumpkin SRS is generated manually at the moment, only up to a large enough size for tests
-# If tests require more points, the parameter can be increased here.
-cmake --build . --parallel --target grumpkin_srs_gen
-./bin/grumpkin_srs_gen 8192
-echo "Generated Grumpkin SRS successfully"
-cd ../
+cmake --build --preset $PRESET --target bb
+
+if [ ! -d ./srs_db/grumpkin ]; then
+  # The Grumpkin SRS is generated manually at the moment, only up to a large enough size for tests
+  # If tests require more points, the parameter can be increased here.
+  (cd ./build && cmake --build . --parallel --target grumpkin_srs_gen && ./bin/grumpkin_srs_gen 8192)
+fi
 
 # Install wasi-sdk.
 ./scripts/install-wasi-sdk.sh

barretenberg/cpp/scripts/install-wasi-sdk.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
 set -eu
 
+if [[ -d ./src/wasi-sdk-20.0 && -d ./src/wasi-sdk-20.0+threads ]]; then
+  echo "WASI already installed. Skipping."
+  exit 0
+fi
+
 # Clean.
 rm -rf ./src/wasi-sdk-*
 
@@ -17,15 +22,15 @@ fi
 # Install the webassembly toolchain.
 mkdir -p src
 cd ./src
-curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-$OS.tar.gz | tar zxfv -
+curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-$OS.tar.gz | tar zxf -
 if [ "$OS" == "linux" ]; then
   # Exceptional linux case that provides an llvm build that works on Ubuntu 20.
-  curl -s -L https://wasi-sdk.s3.eu-west-2.amazonaws.com/yamt-wasi-sdk-20.0.threads.tgz | tar zxfv -
+  curl -s -L https://wasi-sdk.s3.eu-west-2.amazonaws.com/yamt-wasi-sdk-20.0.threads.tgz | tar zxf -
 else
   # For other operating systems, first download the standard release (this is to get the llvm build).
-  curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20%2Bthreads/wasi-sdk-20.0.threads-$OS.tar.gz | tar zxfv -
+  curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20%2Bthreads/wasi-sdk-20.0.threads-$OS.tar.gz | tar zxf -
   # Replace wasi-sysroot in wasi-sdk-20.0+threads with our custom build.
   # It contains libc++ and a patch by yamt to improve thread join stability.
   # Can remove once future releases are more stable.
-  curl -s -L https://wasi-sdk.s3.eu-west-2.amazonaws.com/yamt-wasi-sysroot-20.0.threads.tgz | tar zxfv -
+  curl -s -L https://wasi-sdk.s3.eu-west-2.amazonaws.com/yamt-wasi-sysroot-20.0.threads.tgz | tar zxf -
 fi

barretenberg/ts/bootstrap.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -eu
+
+cd "$(dirname "$0")"
+
+CMD=${1:-}
+
+if [ -n "$CMD" ]; then
+  if [ "$CMD" = "clean" ]; then
+    git clean -fdx
+    exit 0
+  else
+    echo "Unknown command: $CMD"
+    exit 1
+  fi
+fi
+
+yarn install --immutable
+yarn build
+
+# Make bin globally available.
+npm link

barretenberg/ts/package.json

@@ -12,9 +12,7 @@
       "default": "./dest/node/index.js"
     }
   },
-  "bin": {
-    "bb.js": "./dest/node/main.js"
-  },
+  "bin": "./dest/node/main.js",
   "files": [
     "src/",
     "dest/",

barretenberg/ts/scripts/cjs_postprocess.sh

@@ -10,5 +10,5 @@ DIR="./dest/node-cjs"
 # Iterate over all .js files in the directory
 for FILE in $(find "$DIR" -name "*.js"); do
     # Use sed to replace 'import.meta.url' with '""'
-    sed -i 's/import\.meta\.url/""/g' "$FILE"
+    sed -i.bak 's/import\.meta\.url/""/g' "$FILE" && rm "$FILE.bak"
 done

bootstrap.sh

@@ -8,14 +8,10 @@
 #   ./bootstrap.sh clean
 set -eu
 
-export CMD=${1:-}
-
 cd "$(dirname "$0")"
 
-# Bump this number to force a full bootstrap.
-VERSION=1
+CMD=${1:-}
 
-# Remove all untracked files and directories.
 if [ -n "$CMD" ]; then
   if [ "$CMD" = "clean" ]; then
     echo "WARNING: This will erase *all* untracked files, including hooks and submodules."
@@ -24,55 +20,51 @@ if [ -n "$CMD" ]; then
     if [ "$user_input" != "y" ] && [ "$user_input" != "Y" ]; then
       exit 1
     fi
-    rm -f .bootstrapped
+
+    # Remove hooks and submodules.
     rm -rf .git/hooks/*
     rm -rf .git/modules/*
-    git clean -fd
     for SUBMODULE in $(git config --file .gitmodules --get-regexp path | awk '{print $2}'); do
       rm -rf $SUBMODULE
     done
+
+    # Remove all untracked files, directories, nested repos, and .gitignore files.
+    git clean -ffdx
+
+    exit 0
   else
-    echo "Unknown command: $CLEAN"
+    echo "Unknown command: $CMD"
     exit 1
   fi
 fi
 
-if [ ! -f ~/.nvm/nvm.sh ]; then
-  echo "Nvm not found at ~/.nvm"
-  exit 1
-fi
+# if [ ! -f ~/.nvm/nvm.sh ]; then
+#   echo "Nvm not found at ~/.nvm"
+#   exit 1
+# fi
 
 # Install pre-commit git hooks.
 HOOKS_DIR=$(git rev-parse --git-path hooks)
 echo "(cd barretenberg/cpp && ./format.sh staged)" > $HOOKS_DIR/pre-commit
 echo "(cd circuits/cpp && ./format.sh staged)" >> $HOOKS_DIR/pre-commit
-# TODO: Call cci_gen to ensure .circleci/config.yml is up-to-date!
 chmod +x $HOOKS_DIR/pre-commit
 
 git submodule update --init --recursive
 
-# Lightweight bootstrap. Run `./bootstrap.sh clean` to bypass.
-# TODO: We shouldn't do this here. We should call each projects bootstrap script and it should decide between light/heavy.
-if [[ -f .bootstrapped && $(cat .bootstrapped) -eq "$VERSION" ]]; then
-  echo -e '\033[1mRebuild L1 contracts...\033[0m'
-  (cd l1-contracts && .foundry/bin/forge build)
+PROJECTS=(
+  barretenberg
+  noir
+  circuits/cpp
+  l1-contracts
+  yarn-project
+)
 
-  echo -e '\n\033[1mUpdate npm deps...\033[0m'
-  (cd yarn-project && yarn install)
-
-  echo -e '\n\033[1mRebuild Noir contracts...\033[0m'
-  (cd yarn-project/noir-contracts && yarn noir:build:all 2> /dev/null)
-
-  echo -e '\n\033[1mRebuild barretenberg wasm...\033[0m'
-  (cd barretenberg/cpp && cmake --build --preset default && cmake --build --preset wasm && cmake --build --preset wasm-threads)
-
-  echo -e '\n\033[1mRebuild circuits wasm...\033[0m'
-  (cd circuits/cpp && cmake --build --preset wasm -j --target aztec3-circuits.wasm)
-else
-  # Heavy bootstrap.
-  barretenberg/bootstrap.sh
-  circuits/cpp/bootstrap.sh
-  yarn-project/bootstrap.sh
-
-  echo $VERSION > .bootstrapped
-fi
+for P in "${PROJECTS[@]}"; do
+  echo "**************************************"
+  echo -e "\033[1mBootstrapping $P...\033[0m"
+  echo "**************************************"
+  echo
+  $P/bootstrap.sh
+  echo
+  echo
+done

bootstrap/Dockerfile.alpine

@@ -0,0 +1,25 @@
+# At present this doesn't work, as we don't have an alpine wasm toolchain we can use (or it's not directly obvious).
+# Keeping this around though, as it would be nice to resolve this in the future.
+FROM alpine:latest
+RUN apk update && apk add --no-cache \
+  build-base \
+  curl \
+  git \
+  cmake \
+  lsb-release \
+  wget \
+  gnupg \
+  ninja \
+  npm \
+  pkgconf \
+  openssl-dev \
+  jq \
+  llvm \
+  clang \
+  rust \
+  cargo \
+  bash
+RUN npm install -g yarn
+WORKDIR /usr/src
+COPY . .
+RUN ./bootstrap.sh

bootstrap/Dockerfile.lunar

@@ -0,0 +1,23 @@
+FROM ubuntu:lunar
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt update && apt install -y \
+  build-essential \
+  curl \
+  git \
+  cmake \
+  lsb-release \
+  wget \
+  software-properties-common \
+  gnupg \
+  ninja-build \
+  npm \
+  pkg-config \
+  libssl-dev \
+  jq
+RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+RUN npm install -g yarn
+WORKDIR /usr/src
+COPY . .
+RUN ./bootstrap.sh

bootstrap/bootstrap_test.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# This script takes the state of your current repository, and clones it inside of a docker container.
+# You likely don't have a fresh clone, and it's paramount that to test bootstrapping, we don't have any
+# intermediate build state in the context.
+# To achieve this we mount your working directory into the container, and then perform the clone into the container.
+# After cloning the repo we build the relevant dockerfile to bootstrap.
+# "docker-in-docker" is achieved by mounting the host systems docker socket into the container.
+
+DOCKERFILE=${1:-Dockerfile.lunar}
+
+docker build -t bootstrap-build - <<EOF
+FROM ubuntu:latest
+RUN apt update && apt install -y git rsync docker.io
+EOF
+
+docker run -ti --rm -v/run/user/$UID/docker.sock:/var/run/docker.sock -v$(git rev-parse --show-toplevel):/repo:ro bootstrap-build /bin/bash -c "
+# Checkout head.
+mkdir /project && cd /project
+git init
+git remote add origin /repo
+git fetch --depth 1 origin HEAD
+git checkout FETCH_HEAD
+
+# Copy untracked and modified files, and remove deleted files, from our current repo.
+cd /repo
+{ git ls-files --others --exclude-standard ; git diff --name-only --diff-filter=TMAR HEAD ; } | rsync -a --files-from=- . /project
+for F in \$(git ls-files --deleted); do rm /project/\$F > /dev/null 2>&1; done
+
+cd /project
+docker build -f bootstrap/$DOCKERFILE .
+"