diff --git a/.circleci/config.yml b/.circleci/config.yml index e73ac138f1c..fc0c098126e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -114,6 +114,17 @@ jobs: name: "Build" command: cond_spot_run_build noir-packages 32 + noir-acir-tests: + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small + steps: + - *checkout + - *setup_env + - run: + name: "Build" + command: cond_spot_run_build noir-acir-tests 32 + # Barretenberg barretenberg-wasm-linux-clang: docker: @@ -993,6 +1004,10 @@ workflows: - noir-arm64 <<: *defaults - noir-packages: *defaults + - noir-acir-tests: + requires: + - noir-ecr-manifest + <<: *defaults # Barretenberg - barretenberg-x86_64-linux-gcc: *defaults @@ -1011,6 +1026,7 @@ workflows: - barretenberg-acir-tests-bb: requires: - barretenberg-x86_64-linux-clang-assert + - noir-acir-tests <<: *defaults - bb-js: requires: @@ -1023,6 +1039,7 @@ workflows: - bb-js-acir-tests: requires: - bb-js + - noir-acir-tests <<: *defaults - l1-contracts: *defaults diff --git a/barretenberg/acir_tests/Dockerfile.bb b/barretenberg/acir_tests/Dockerfile.bb index 12ca9f84fd2..0fda0d6ec15 100644 --- a/barretenberg/acir_tests/Dockerfile.bb +++ b/barretenberg/acir_tests/Dockerfile.bb @@ -1,8 +1,10 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-acir-tests as noir-acir-tests FROM node:18-alpine RUN apk update && apk add git bash curl jq coreutils COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build +COPY --from=noir-acir-tests /usr/src/noir/tooling/nargo_cli/tests /usr/src/noir/tooling/nargo_cli/tests WORKDIR /usr/src/barretenberg/acir_tests COPY . . # Run every acir test through native bb build prove_then_verify flow. diff --git a/barretenberg/acir_tests/Dockerfile.bb.js b/barretenberg/acir_tests/Dockerfile.bb.js index de31aff9b05..3e4ddfb91e3 100644 --- a/barretenberg/acir_tests/Dockerfile.bb.js +++ b/barretenberg/acir_tests/Dockerfile.bb.js @@ -1,7 +1,9 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-acir-tests as noir-acir-tests FROM node:18 COPY --from=0 /usr/src/barretenberg/ts /usr/src/barretenberg/ts +COPY --from=noir-acir-tests /usr/src/noir/tooling/nargo_cli/tests /usr/src/noir/tooling/nargo_cli/tests RUN apt update && apt install -y lsof jq WORKDIR /usr/src/barretenberg/acir_tests # Build/install ts apps. diff --git a/barretenberg/acir_tests/Dockerfile.bb.sol b/barretenberg/acir_tests/Dockerfile.bb.sol index bbe4a8a03cd..1cbc3ce8a22 100644 --- a/barretenberg/acir_tests/Dockerfile.bb.sol +++ b/barretenberg/acir_tests/Dockerfile.bb.sol @@ -1,13 +1,15 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-sol +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-acir-tests as noir-acir-tests FROM node:18-alpine RUN apk update && apk add git bash curl jq COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build COPY --from=1 /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol +COPY --from=noir-acir-tests /usr/src/noir/tooling/nargo_cli/tests /usr/src/noir/tooling/nargo_cli/tests COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/anvil /usr/local/bin/anvil WORKDIR /usr/src/barretenberg/acir_tests COPY . . -# Run every acir test through a solidity verifier". +# Run every acir test through a solidity verifier. RUN (cd sol-test && yarn) RUN PARALLEL=1 FLOW=sol ./run_acir_tests.sh diff --git a/barretenberg/acir_tests/Dockerfile.noir_acir_tests b/barretenberg/acir_tests/Dockerfile.noir_acir_tests new file mode 100644 index 00000000000..938e55ca75e --- /dev/null +++ b/barretenberg/acir_tests/Dockerfile.noir_acir_tests @@ -0,0 +1,10 @@ +# This is a slightly odd location for this Dockerfile, as ostensibly it should belong next to the tests themselves. +# However, the noir subrepo has no concept of the aztec build pipeline, so the Dockerfile wouldn't make sense there. +# So, it lives here. +# This chains off the nargo build, and creates a container with a compiled set of acir tests. +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir +RUN apk add bash jq +ENV PATH="/usr/src/noir/target/release:${PATH}" +WORKDIR /usr/src/noir/tooling/nargo_cli/tests +COPY . . +RUN ./rebuild.sh \ No newline at end of file diff --git a/barretenberg/acir_tests/clone_test_vectors.sh b/barretenberg/acir_tests/clone_test_vectors.sh index c5023369990..e2d4c34e2b7 100755 --- a/barretenberg/acir_tests/clone_test_vectors.sh +++ b/barretenberg/acir_tests/clone_test_vectors.sh @@ -1,19 +1,8 @@ #!/bin/bash set -eu -# Pull down the test vectors from the noir repo, if we don't have the folder already. +TEST_SRC=${TEST_SRC:-../../noir/tooling/nargo_cli/tests/acir_artifacts} + if [ ! -d acir_tests ]; then - if [ -n "${TEST_SRC:-}" ]; then - cp -R $TEST_SRC acir_tests - else - rm -rf noir - git clone -b $BRANCH --filter=blob:none --no-checkout https://github.com/noir-lang/noir.git - cd noir - git sparse-checkout init --cone - git sparse-checkout set test_programs/acir_artifacts - git checkout - cd .. - mv noir/test_programs/acir_artifacts acir_tests - rm -rf noir - fi -fi + cp -R $TEST_SRC acir_tests +fi \ No newline at end of file diff --git a/build_manifest.yml b/build_manifest.yml index 97b8124322e..6244f89393d 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -1,3 +1,21 @@ +noir: + # Add less aggressive rebuildPatterns. + buildDir: noir + +noir-packages: + buildDir: noir + dockerfile: Dockerfile.packages + +noir-acir-tests: + buildDir: noir/tooling/nargo_cli/tests + # Awkward. Maybe change dockerfile paths to be absolute. + dockerfile: ../../../../barretenberg/acir_tests/Dockerfile.noir_acir_tests + rebuildPatterns: + - ^noir/tooling/nargo_cli/tests + - ^barretenberg/acir_tests/Dockerfile.noir_acir_tests + dependencies: + - noir + barretenberg-x86_64-linux-clang: buildDir: barretenberg/cpp dockerfile: dockerfiles/Dockerfile.x86_64-linux-clang @@ -38,6 +56,7 @@ barretenberg-acir-tests-bb: dockerfile: Dockerfile.bb dependencies: - barretenberg-x86_64-linux-clang-assert + - noir-acir-tests barretenberg-acir-tests-bb-sol: buildDir: barretenberg/acir_tests @@ -45,12 +64,14 @@ barretenberg-acir-tests-bb-sol: dependencies: - barretenberg-x86_64-linux-clang-assert - barretenberg-x86_64-linux-clang-sol + - noir-acir-tests barretenberg-acir-tests-bb.js: buildDir: barretenberg/acir_tests dockerfile: Dockerfile.bb.js dependencies: - bb.js + - noir-acir-tests l1-contracts: buildDir: l1-contracts @@ -149,10 +170,3 @@ docs: - ^.*.nr$ dependencies: - yarn-project - -noir: - buildDir: noir - -noir-packages: - buildDir: noir - dockerfile: Dockerfile.packages diff --git a/noir/tooling/nargo_cli/tests/rebuild.sh b/noir/tooling/nargo_cli/tests/rebuild.sh index d89402fb1cf..dfc3dc5c967 100755 --- a/noir/tooling/nargo_cli/tests/rebuild.sh +++ b/noir/tooling/nargo_cli/tests/rebuild.sh @@ -1,56 +1,72 @@ #!/bin/bash set -e -excluded_dirs=("workspace" "workspace_default_member") +process_dir() { + local dir=$1 + local current_dir=$2 + local dir_name=$(basename "$dir") + + if [[ ! -d "$current_dir/acir_artifacts/$dir_name" ]]; then + mkdir -p $current_dir/acir_artifacts/$dir_name + fi + + cd $dir + if [ -d ./target/ ]; then + rm -r ./target/ + fi + nargo compile && nargo execute witness + + if [ -f ./target/witness.tr ]; then + mv ./target/witness.tr ./target/witness.gz + fi + + if [ -f ./target/${dir_name}.json ]; then + jq -r '.bytecode' ./target/${dir_name}.json | base64 -d > ./target/acir.gz + fi + + rm ./target/${dir_name}.json + + if [ -d "$current_dir/acir_artifacts/$dir_name/target" ]; then + rm -r "$current_dir/acir_artifacts/$dir_name/target" + fi + mkdir $current_dir/acir_artifacts/$dir_name/target + + mv ./target/*.gz $current_dir/acir_artifacts/$dir_name/target/ + cd $current_dir +} + +export -f process_dir + +excluded_dirs=("workspace" "workspace_default_member") current_dir=$(pwd) base_path="$current_dir/execution_success" -# Clear the acir_artifacts directory of any existing artifacts rm -rf $current_dir/acir_artifacts mkdir -p $current_dir/acir_artifacts -# Loop over every directory +# Gather directories to process. +dirs_to_process=() for dir in $base_path/*; do - if [[ ! -d $dir ]]; then - continue - fi - - dir_name=$(basename "$dir") - - if [[ ! " ${excluded_dirs[@]} " =~ " ${dir_name} " ]]; then - if [[ ! -d "$current_dir/acir_artifacts/$dir_name" ]]; then - mkdir -p $current_dir/acir_artifacts/$dir_name - fi - - cd $dir - if [ -d ./target/ ]; then - rm -r ./target/ - fi - nargo compile && nargo execute witness - - # Rename witness.tr to witness.gz - if [ -f ./target/witness.tr ]; then - mv ./target/witness.tr ./target/witness.gz - fi - - # Extract bytecode field from JSON, base64 decode it, and save it to the target directory - if [ -f ./target/${dir_name}.json ]; then - jq -r '.bytecode' ./target/${dir_name}.json | base64 -d > ./target/acir.gz - fi - - # Delete the JSON file after extracting bytecode field - rm ./target/${dir_name}.json - - # Clear the target directory in acir_artifacts - if [ -d "$current_dir/acir_artifacts/$dir_name/target" ]; then - rm -r "$current_dir/acir_artifacts/$dir_name/target" - fi - mkdir $current_dir/acir_artifacts/$dir_name/target - - # Move the artifacts from the target directory to the corresponding directory in acir_artifacts - mv ./target/*.gz $current_dir/acir_artifacts/$dir_name/target/ - - cd $base_path - fi + if [[ ! -d $dir ]] || [[ " ${excluded_dirs[@]} " =~ " $(basename "$dir") " ]]; then + continue + fi + dirs_to_process+=("$dir") +done + +# Process each directory in parallel +pids=() +for dir in "${dirs_to_process[@]}"; do + process_dir "$dir" "$current_dir" & + pids+=($!) +done + +# Check the exit status of each background job. +for pid in "${pids[@]}"; do + wait $pid || exit_status=$? done + +# Exit with a failure status if any job failed. +if [ ! -z "$exit_status" ]; then + exit $exit_status +fi \ No newline at end of file