feat: Integrate new proving systems in e2e (#6971)

**THE AWESOME:** PXE uses ClientIVC. Public kernels, rollup, and parity circuits use vanilla recursion-based IVC using UltraHonk. 🎉 Linear time proofs without FFTs, better constraint counts for poseidon (eventually) and no recursion in the private kernel circuits **THE CAVEATS:** This merges with many TODOs to reinstate functionality, improve security and performance. - If you are lead to believe honk proofs are constant-sized, you have been fed lies We are rounding up to a proof size that allows 2**28 gates. Please take effort to NOT rely on this assumption, as it is just a stop-gap. It is OK to temporarily rely on this to simplify kernel development in particular, but we should not hard-code this assumption into e.g. typescript in the future - Things are a bit slower now, not faster We prioritized correctness and any slowdowns will be investigated soon. Some of it comes from the larger constant size proof. As well, we will soon be able to drop constraint count in poisedon with UH - Transcript hashing is not yet in place - #7410 Alvaro's recent work with the VK tree had to be partially undone (aka the parts that made it really validate in kernels, so most of it for now) because our tube VK currently cannot be written ahead of time. What's more, correctly handling log-sized proofs will mean many tube and kernel VKs in the future. This will need a pass. - #7374 This test used a lot of mocking and was stubbed out in the interest of time. It's not necessarily hard to bring back. - #7373 We need one soon. The relevant deploying functionality has been commented out for now with this issue linked. - #7372 This needs to be double-checked for current kernel proof, but we will do this post-merge with a pass on observability - #7371 We need to figure out how to get this program stack bincode-encoded in the future - #7370 Right now redundant VKs are serialized with the client ivc proofs - #7369 Related to #7410, currently tube proving is too unlike the rest of the pattern - #7368 It's currently a bit of a mess of grabbing bytecode artifacts with ad-hoc logic - AztecProtocol/barretenberg#1048 Inputs should be passed to the tube circuit correctly, but they are NOT currently used. Fixing this should be contained within bb --------- Co-authored-by: codygunton <codygunton@gmail.com> Co-authored-by: ledwards2225 <l.edwards.d@gmail.com> Co-authored-by: lucasxia01 <lucasxia01@gmail.com> Co-authored-by: maramihali <mara@aztecprotocol.com>
AztecProtocol · Jul 11, 2024 · 723a0c1 · 723a0c1
1 parent a6a605d
commit 723a0c1
Show file tree

Hide file tree

Showing 106 changed files with 1,856 additions and 827 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -157,7 +157,7 @@ jobs:
       - uses: ./.github/ci-setup-action
       - name: Setup and Test
         uses: ./.github/ensure-tester-with-images
-        timeout-minutes: 40
+        timeout-minutes: 45
         with:
           runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
           builder_type: builder-x86
@@ -166,6 +166,7 @@ jobs:
           builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
           # command to produce the images in case they don't exist
           builder_command: cd yarn-project/end-to-end/ && ../../scripts/earthly-ci +${{ matrix.test }}
+          tester_ttl: 40
           run: |
             set -eux
             cd ./yarn-project/end-to-end/
@@ -381,7 +382,7 @@ jobs:
       - name: "Format noir-projects"
         working-directory: ./noir-projects/
         timeout-minutes: 40
-        run: | 
+        run: |
           earthly-ci --no-output \
           --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
           --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
@@ -465,7 +466,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
-      # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: yarn-project-formatting-x86
@@ -474,13 +474,11 @@ jobs:
         run: earthly-ci --no-output ./yarn-project/+format-check
 
   yarn-project-test:
-    # quiet machine
     needs: build
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
-      # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: yarn-project-test-x86
@@ -527,7 +525,7 @@ jobs:
       - name: "Docs Preview"
         if: github.event.number
         timeout-minutes: 40
-        run: | 
+        run: |
           earthly-ci --no-output \
           --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
           --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
@@ -639,7 +637,7 @@ jobs:
           earthly-ci \
           --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
           --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-          --artifact +gates-report/gates_report.json 
+          --artifact +gates-report/gates_report.json
           mv gates_report.json ../protocol_circuits_report.json
 
       - name: Compare gates reports

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -5,14 +5,19 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "address": "${config:mainframeAddress}",
-      "port": 9221,
+      "port": 9229,
       "type": "node",
       "request": "attach",
       "name": "Attach to Remote",
       "skipFiles": ["<node_internals>/**"],
       "localRoot": "${workspaceFolder}",
       "sourceMaps": true
     },
+    {
+      "type": "node",
+      "request": "attach",
+      "name": "Attach",
+      "port": 9229
+    }
   ]
 }
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -120,7 +120,7 @@
     "editor.defaultFormatter": "hashicorp.terraform"
   },
   "[typescript]": {
-    "editor.defaultFormatter": "esbenp.prettier-vscode"
+    "editor.defaultFormatter": "vscode.typescript-language-features"
   },
   "[javascript]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"

diff --git a/barretenberg/.vscode/launch.json b/barretenberg/.vscode/launch.json
@@ -0,0 +1,20 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "LLDB: Prove Ultra Honk Output All",
+            "type": "lldb",
+            "request": "launch",
+            "program": "/mnt/user-data/adam/aztec-packages/barretenberg/cpp/build-debug/bin/bb",
+            "args": [
+                "prove_ultra_honk_output_all",
+                "-o", "/tmp/bb-095XQV/tmp-PPXk0y",
+                "-b", "/tmp/bb-095XQV/tmp-PPXk0y/PrivateKernelInitArtifact-bytecode",
+                "-w", "/tmp/bb-095XQV/tmp-PPXk0y/witness.gz",
+                "-v"
+            ],
+            "cwd": "${workspaceFolder}",
+            "stopOnEntry": false
+        }
+    ]
+}
diff --git a/barretenberg/acir_tests/reset_acir_tests.sh b/barretenberg/acir_tests/reset_acir_tests.sh
@@ -1,4 +1,4 @@
-# Run from barretenberg/acir_tests
+# Run from within barretenberg/acir_tests
 cd ../../noir/noir-repo
 cargo clean
 noirup -p .

diff --git a/barretenberg/cpp/CMakeLists.txt b/barretenberg/cpp/CMakeLists.txt
@@ -143,6 +143,16 @@ include(cmake/gtest.cmake)
 include(cmake/benchmark.cmake)
 include(cmake/module.cmake)
 include(cmake/msgpack.cmake)
+
+# We do not need to bloat barretenberg.wasm with gzip functionality in a browser context as the browser can do this
+if (NOT WASM)
+    include(cmake/libdeflate.cmake)
+else()
+    set(DISABLE_AZTEC_VM ON)
+endif()
+if(DISABLE_AZTEC_VM)
+    add_definitions(-DDISABLE_AZTEC_VM=1)
+endif()
 include(cmake/backward-cpp.cmake)
 
 if (WASM)

diff --git a/barretenberg/cpp/cmake/libdeflate.cmake b/barretenberg/cpp/cmake/libdeflate.cmake
@@ -0,0 +1,10 @@
+include(FetchContent)
+
+FetchContent_Declare(
+  libdeflate
+  GIT_REPOSITORY https://github.com/ebiggers/libdeflate.git
+  GIT_TAG        b03254d978d7af21a7512dee8fdc3367bc15c656
+)
+
+# Download and populate libdeflate
+FetchContent_MakeAvailable(libdeflate)
diff --git a/barretenberg/cpp/scripts/flamegraph.pl b/barretenberg/cpp/scripts/flamegraph.pl
@@ -410,7 +410,7 @@ sub color {
 		} elsif ($name =~ m:^L?(java|javax|jdk|net|org|com|io|sun)/:) {	# Java
 			$type = "green";
 		} elsif ($name =~ /:::/) {      # Java, typical perf-map-agent method separator
-			$type = "green";	              
+			$type = "green";
 		} elsif ($name =~ /::/) {	# C++
 			$type = "yellow";
 		} elsif ($name =~ m:_\[k\]$:) {	# kernel annotation
@@ -818,7 +818,7 @@ sub flow {
 	// ctrl-F for search
 	// ctrl-I to toggle case-sensitive search
 	window.addEventListener("keydown",function (e) {
-		if (e.keyCode === 114 || (e.ctrlKey && e.keyCode === 70)) {
+		if (e.keyCode === 103 || (e.ctrlKey && e.keyCode === 70)) {
 			e.preventDefault();
 			search_prompt();
 		}

diff --git a/barretenberg/cpp/src/barretenberg/bb/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/bb/CMakeLists.txt
@@ -1,4 +1,4 @@
-if (NOT(FUZZING))
+if (NOT(FUZZING) AND NOT(WASM))
     add_executable(
         bb
         main.cpp
@@ -11,6 +11,8 @@ if (NOT(FUZZING))
         PRIVATE
         barretenberg
         env
+        circuit_checker
+        libdeflate::libdeflate_static
     )
     if(CHECK_CIRCUIT_STACKTRACES)
         target_link_libraries(