feat: folding acir programs (#6685)

Introduces (DISABLED) tests of ClientIvc accumulation for a stack of
acir-represented circuits generated by noir. Tests appear in three
locations: the bb acir tests, the bb.js acir tests, and the new
AcirIntegration test suite (which facilitates a better debugging
workflow).

---------

Co-authored-by: codygunton <codygunton@gmail.com>

barretenberg/Earthfile

@@ -38,6 +38,8 @@ barretenberg-acir-tests-bb:
     RUN FLOW=prove_and_verify_mega_honk ./run_acir_tests.sh 6_array
     # Construct and verify a UltraHonk proof for all ACIR programs using the new witness stack workflow
     RUN FLOW=prove_and_verify_ultra_honk_program ./run_acir_tests.sh
+    # Fold and verify an ACIR program stack using ClientIvc
+    RUN FLOW=fold_and_verify_program ./run_acir_tests.sh fold_basic
     # This is a "full" Goblin flow. It constructs and verifies four proofs: MegaHonk, ECCVM, Translator, and merge
     RUN FLOW=prove_and_verify_goblin ./run_acir_tests.sh 6_array
     # Run 1_mul through native bb build, all_cmds flow, to test all cli args.

barretenberg/acir_tests/Dockerfile.bb.js

@@ -21,10 +21,12 @@ RUN BIN=../ts/dest/node/main.js FLOW=prove_then_verify ./run_acir_tests.sh ecdsa
 RUN BIN=../ts/dest/node/main.js FLOW=prove_then_verify_ultra_honk ./run_acir_tests.sh nested_array_dynamic
 # Run a single arbitrary test not involving recursion through bb.js for Plonk
 RUN BIN=../ts/dest/node/main.js FLOW=prove_and_verify ./run_acir_tests.sh poseidon_bn254_hash
-# Run a single arbitrary test not involving recursion through bb.js for MegaHonk
+# Run a single arbitrary test not involving recursion through bb.js for UltraHonk
 RUN BIN=../ts/dest/node/main.js FLOW=prove_and_verify_ultra_honk ./run_acir_tests.sh closures_mut_ref
-# Run a single arbitrary test for separate prove and verify for UltraHonk
+# Run a single arbitrary test for separate prove and verify for MegaHonk
 RUN BIN=../ts/dest/node/main.js FLOW=prove_and_verify_mega_honk ./run_acir_tests.sh 6_array
+# Fold and verify an ACIR program stack
+RUN BIN=../ts/dest/node/main.js FLOW=fold_and_verify_program ./run_acir_tests.sh fold_basic
 # Run a single arbitrary test not involving recursion through bb.js for full Goblin
 RUN BIN=../ts/dest/node/main.js FLOW=prove_and_verify_goblin ./run_acir_tests.sh 6_array
 # Run 1_mul through bb.js build, all_cmds flow, to test all cli args.

barretenberg/acir_tests/flows/fold_and_verify_program.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+
+VFLAG=${VERBOSE:+-v}
+
+$BIN fold_and_verify_program $VFLAG -c $CRS_PATH -b ./target/program.json

barretenberg/acir_tests/flows/prove_and_verify_mega_honk_program.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+
+VFLAG=${VERBOSE:+-v}
+
+$BIN prove_and_verify_mega_honk_program $VFLAG -c $CRS_PATH -b ./target/program.json

barretenberg/cpp/src/CMakeLists.txt

@@ -104,6 +104,7 @@ include(GNUInstallDirs)
 message(STATUS "Compiling all-in-one barretenberg archive")
 
 set(BARRETENBERG_TARGET_OBJECTS
+    $<TARGET_OBJECTS:client_ivc_objects>
     $<TARGET_OBJECTS:commitment_schemes_objects>
     $<TARGET_OBJECTS:common_objects>
     $<TARGET_OBJECTS:client_ivc_objects>

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -1,4 +1,5 @@
 #include "barretenberg/bb/file_io.hpp"
+#include "barretenberg/client_ivc/client_ivc.hpp"
 #include "barretenberg/common/serialize.hpp"
 #include "barretenberg/dsl/acir_format/acir_format.hpp"
 #include "barretenberg/dsl/types.hpp"
@@ -219,6 +220,34 @@ bool proveAndVerifyHonkProgram(const std::string& bytecodePath, const std::strin
     return true;
 }
 
+bool foldAndVerifyProgram(const std::string& bytecodePath, const std::string& witnessPath)
+{
+    using Flavor = MegaFlavor; // This is the only option
+    using Builder = Flavor::CircuitBuilder;
+
+    init_bn254_crs(1 << 18);
+    init_grumpkin_crs(1 << 14);
+
+    ClientIVC ivc;
+    ivc.structured_flag = true;
+
+    auto program_stack = acir_format::get_acir_program_stack(bytecodePath, witnessPath);
+
+    // Accumulate the entire program stack into the IVC
+    while (!program_stack.empty()) {
+        auto stack_item = program_stack.back();
+
+        // Construct a bberg circuit from the acir representation
+        auto circuit = acir_format::create_circuit<Builder>(
+            stack_item.constraints, 0, stack_item.witness, false, ivc.goblin.op_queue);
+
+        ivc.accumulate(circuit);
+
+        program_stack.pop_back();
+    }
+    return ivc.prove_and_verify();
+}
+
 /**
  * @brief Proves and Verifies an ACIR circuit
  *
@@ -832,6 +861,12 @@ int main(int argc, char* argv[])
         if (command == "prove_and_verify_ultra_honk_program") {
             return proveAndVerifyHonkProgram<UltraFlavor>(bytecode_path, witness_path) ? 0 : 1;
         }
+        if (command == "prove_and_verify_mega_honk_program") {
+            return proveAndVerifyHonkProgram<MegaFlavor>(bytecode_path, witness_path) ? 0 : 1;
+        }
+        if (command == "fold_and_verify_program") {
+            return foldAndVerifyProgram(bytecode_path, witness_path) ? 0 : 1;
+        }
         if (command == "prove_and_verify_goblin") {
             return proveAndVerifyGoblin(bytecode_path, witness_path) ? 0 : 1;
         }

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -117,4 +117,18 @@ std::vector<std::shared_ptr<ClientIVC::VerificationKey>> ClientIVC::precompute_f
     return vkeys;
 }
 
+/**
+ * @brief Construct and verify a proof for the IVC
+ * @note Use of this method only makes sense when the prover and verifier are the same entity, e.g. in
+ * development/testing.
+ *
+ */
+bool ClientIVC::prove_and_verify()
+{
+    auto proof = prove();
+
+    auto verifier_inst = std::make_shared<VerifierInstance>(this->instance_vk);
+    return verify(proof, { this->verifier_accumulator, verifier_inst });
+}
+
 } // namespace bb

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp

@@ -87,6 +87,8 @@ class ClientIVC {
 
     bool verify(Proof& proof, const std::vector<std::shared_ptr<VerifierInstance>>& verifier_instances);
 
+    bool prove_and_verify();
+
     HonkProof decider_prove() const;
 
     std::vector<std::shared_ptr<VerificationKey>> precompute_folding_verification_keys(std::vector<ClientCircuit>);

barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt

@@ -2,6 +2,7 @@ barretenberg_module(
     dsl
     plonk
     ultra_honk
+    client_ivc
     stdlib_sha256
     stdlib_aes128
     stdlib_keccak

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.cpp

@@ -340,7 +340,8 @@ template <>
 UltraCircuitBuilder create_circuit(const AcirFormat& constraint_system,
                                    size_t size_hint,
                                    WitnessVector const& witness,
-                                   bool honk_recursion)
+                                   bool honk_recursion,
+                                   [[maybe_unused]] std::shared_ptr<ECCOpQueue>)
 {
     Builder builder{
         size_hint, witness, constraint_system.public_inputs, constraint_system.varnum, constraint_system.recursive
@@ -365,10 +366,10 @@ template <>
 MegaCircuitBuilder create_circuit(const AcirFormat& constraint_system,
                                   [[maybe_unused]] size_t size_hint,
                                   WitnessVector const& witness,
-                                  bool honk_recursion)
+                                  bool honk_recursion,
+                                  std::shared_ptr<ECCOpQueue> op_queue)
 {
     // Construct a builder using the witness and public input data from acir and with the goblin-owned op_queue
-    auto op_queue = std::make_shared<ECCOpQueue>(); // instantiate empty op_queue
     auto builder = MegaCircuitBuilder{ op_queue, witness, constraint_system.public_inputs, constraint_system.varnum };
 
     // Populate constraints in the builder via the data in constraint_system

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.hpp

@@ -119,6 +119,11 @@ struct AcirProgramStack {
     std::vector<AcirFormat> constraint_systems;
     WitnessVectorStack witness_stack;
 
+    AcirProgramStack(std::vector<AcirFormat>& constraint_systems_in, WitnessVectorStack& witness_stack_in)
+        : constraint_systems(constraint_systems_in)
+        , witness_stack(witness_stack_in)
+    {}
+
     size_t size() const { return witness_stack.size(); }
     bool empty() const { return witness_stack.empty(); }
 
@@ -138,7 +143,8 @@ template <typename Builder = UltraCircuitBuilder>
 Builder create_circuit(const AcirFormat& constraint_system,
                        size_t size_hint = 0,
                        WitnessVector const& witness = {},
-                       bool honk_recursion = false);
+                       bool honk_recursion = false,
+                       std::shared_ptr<ECCOpQueue> op_queue = std::make_shared<ECCOpQueue>());
 
 template <typename Builder>
 void build_constraints(Builder& builder,

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_integration.test.cpp

@@ -1,3 +1,4 @@
+#include "barretenberg/client_ivc/client_ivc.hpp"
 #ifndef __wasm__
 #include "barretenberg/bb/exec_pipe.hpp"
 #include "barretenberg/common/streams.hpp"
@@ -88,16 +89,49 @@ class AcirIntegrationTest : public ::testing::Test {
         auto verifier = composer.create_verifier(builder);
         return verifier.verify_proof(proof);
     }
-};
 
-class AcirIntegrationSingleTest : public AcirIntegrationTest, public testing::WithParamInterface<std::string> {
+    void add_some_simple_RAM_gates(auto& circuit)
+    {
+        std::array<uint32_t, 3> ram_values{ circuit.add_variable(5),
+                                            circuit.add_variable(10),
+                                            circuit.add_variable(20) };
+
+        size_t ram_id = circuit.create_RAM_array(3);
+
+        for (size_t i = 0; i < 3; ++i) {
+            circuit.init_RAM_element(ram_id, i, ram_values[i]);
+        }
+
+        auto val_idx_1 = circuit.read_RAM_array(ram_id, circuit.add_variable(1));
+        auto val_idx_2 = circuit.read_RAM_array(ram_id, circuit.add_variable(2));
+        auto val_idx_3 = circuit.read_RAM_array(ram_id, circuit.add_variable(0));
+
+        circuit.create_big_add_gate({
+            val_idx_1,
+            val_idx_2,
+            val_idx_3,
+            circuit.zero_idx,
+            1,
+            1,
+            1,
+            0,
+            -35,
+        });
+    }
+
   protected:
     static void SetUpTestSuite() { srs::init_crs_factory("../srs_db/ignition"); }
 };
 
+class AcirIntegrationSingleTest : public AcirIntegrationTest, public testing::WithParamInterface<std::string> {};
+
 class AcirIntegrationFoldingTest : public AcirIntegrationTest, public testing::WithParamInterface<std::string> {
   protected:
-    static void SetUpTestSuite() { srs::init_crs_factory("../srs_db/ignition"); }
+    static void SetUpTestSuite()
+    {
+        srs::init_crs_factory("../srs_db/ignition");
+        srs::init_grumpkin_crs_factory("../srs_db/grumpkin");
+    }
 };
 
 TEST_P(AcirIntegrationSingleTest, DISABLED_ProveAndVerifyProgram)
@@ -350,14 +384,72 @@ TEST_P(AcirIntegrationFoldingTest, DISABLED_ProveAndVerifyProgramStack)
     }
 }
 
+TEST_P(AcirIntegrationFoldingTest, DISABLED_FoldAndVerifyProgramStack)
+{
+    using Flavor = MegaFlavor;
+    using Builder = Flavor::CircuitBuilder;
+
+    std::string test_name = GetParam();
+    auto program_stack = get_program_stack_data_from_test_file(test_name);
+
+    ClientIVC ivc;
+    ivc.structured_flag = true;
+
+    while (!program_stack.empty()) {
+        auto program = program_stack.back();
+
+        // Construct a bberg circuit from the acir representation
+        auto circuit =
+            acir_format::create_circuit<Builder>(program.constraints, 0, program.witness, false, ivc.goblin.op_queue);
+
+        ivc.accumulate(circuit);
+
+        CircuitChecker::check(circuit);
+        // EXPECT_TRUE(prove_and_verify_honk<Flavor>(ivc.prover_instance));
+
+        program_stack.pop_back();
+    }
+
+    EXPECT_TRUE(ivc.prove_and_verify());
+}
+
 INSTANTIATE_TEST_SUITE_P(AcirTests,
                          AcirIntegrationFoldingTest,
-                         testing::Values("fold_after_inlined_calls",
-                                         "fold_basic",
-                                         "fold_basic_nested_call",
-                                         "fold_call_witness_condition",
-                                         "fold_complex_outputs",
-                                         "fold_distinct_return",
-                                         "fold_fibonacci",
-                                         "fold_numeric_generic_poseidon"));
-#endif
+                         testing::Values("fold_basic", "fold_basic_nested_call"));
+
+/**
+ * @brief Ensure that adding gates post-facto to a circuit generated from acir still results in a valid circuit
+ * @details This is a pattern required by e.g. ClientIvc which appends recursive verifiers to acir-generated circuits
+ *
+ */
+TEST_F(AcirIntegrationTest, DISABLED_UpdateAcirCircuit)
+{
+    using Flavor = MegaFlavor;
+    using Builder = Flavor::CircuitBuilder;
+
+    std::string test_name = "6_array"; // arbitrary program with RAM gates
+    auto acir_program = get_program_data_from_test_file(test_name);
+
+    // Construct a bberg circuit from the acir representation
+    auto circuit = acir_format::create_circuit<Builder>(acir_program.constraints, 0, acir_program.witness);
+
+    EXPECT_TRUE(CircuitChecker::check(circuit));
+
+    // Now append some RAM gates onto the circuit generated from acir and confirm that its still valid. (First, check
+    // that the RAM operations constitute a valid independent circuit).
+    {
+        Builder circuit;
+        add_some_simple_RAM_gates(circuit);
+        EXPECT_TRUE(CircuitChecker::check(circuit));
+        EXPECT_TRUE(prove_and_verify_honk<Flavor>(circuit));
+    }
+
+    // Now manually append the simple RAM circuit to the circuit generated from acir
+    add_some_simple_RAM_gates(circuit);
+
+    // Confirm that the result is still valid
+    EXPECT_TRUE(CircuitChecker::check(circuit));
+    EXPECT_TRUE(prove_and_verify_honk<Flavor>(circuit));
+}
+
+#endif

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp

@@ -10,7 +10,7 @@ AcirFormat circuit_buf_to_acir_format(std::vector<uint8_t> const& buf);
  * @brief Converts from the ACIR-native `WitnessMap` format to Barretenberg's internal `WitnessVector` format.
  *
  * @param buf Serialized representation of a `WitnessMap`.
- * @return A `WitnessVector` equivalent to the passed `WitnessMap`.xo
+ * @return A `WitnessVector` equivalent to the passed `WitnessMap`.
  * @note This transformation results in all unassigned witnesses within the `WitnessMap` being assigned the value 0.
  *       Converting the `WitnessVector` back to a `WitnessMap` is unlikely to return the exact same `WitnessMap`.
  */

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp

@@ -1,6 +1,7 @@
 #include "c_bind.hpp"
 #include "../acir_format/acir_to_constraint_buf.hpp"
 #include "acir_composer.hpp"
+#include "barretenberg/client_ivc/client_ivc.hpp"
 #include "barretenberg/common/mem.hpp"
 #include "barretenberg/common/net.hpp"
 #include "barretenberg/common/serialize.hpp"
@@ -78,6 +79,33 @@ WASM_EXPORT void acir_prove_and_verify_ultra_honk(uint8_t const* acir_vec, uint8
     *result = verifier.verify_proof(proof);
 }
 
+WASM_EXPORT void acir_fold_and_verify_program_stack(uint8_t const* acir_vec, uint8_t const* witness_vec, bool* result)
+{
+    using ProgramStack = acir_format::AcirProgramStack;
+    using Builder = MegaCircuitBuilder;
+
+    auto constraint_systems = acir_format::program_buf_to_acir_format(from_buffer<std::vector<uint8_t>>(acir_vec));
+    auto witness_stack = acir_format::witness_buf_to_witness_stack(from_buffer<std::vector<uint8_t>>(witness_vec));
+
+    ProgramStack program_stack{ constraint_systems, witness_stack };
+
+    ClientIVC ivc;
+    ivc.structured_flag = true;
+
+    while (!program_stack.empty()) {
+        auto stack_item = program_stack.back();
+
+        // Construct a bberg circuit from the acir representation
+        auto builder = acir_format::create_circuit<Builder>(
+            stack_item.constraints, 0, stack_item.witness, false, ivc.goblin.op_queue);
+
+        ivc.accumulate(builder);
+
+        program_stack.pop_back();
+    }
+    *result = ivc.prove_and_verify();
+}
+
 WASM_EXPORT void acir_prove_and_verify_mega_honk(uint8_t const* acir_vec, uint8_t const* witness_vec, bool* result)
 {
     auto constraint_system = acir_format::circuit_buf_to_acir_format(from_buffer<std::vector<uint8_t>>(acir_vec));

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.hpp

@@ -49,6 +49,14 @@ WASM_EXPORT void acir_prove_and_verify_mega_honk(uint8_t const* constraint_syste
                                                  uint8_t const* witness_buf,
                                                  bool* result);
 
+/**
+ * @brief Fold and verify a set of circuits using ClientIvc
+ *
+ */
+WASM_EXPORT void acir_fold_and_verify_program_stack(uint8_t const* constraint_system_buf,
+                                                    uint8_t const* witness_buf,
+                                                    bool* result);
+
 /**
  * @brief Construct a full goblin proof
  * @details Makes a call to accumulate to a final circuit before constructing a Goblin proof