feat: split merge into recursive verification and proving (#7801)

ClientIvc/AztecIvc originally had a single method `accumulate(circuit)`
which did two things under the hood: 1) append recursive PG/merge
verifiers, and 2) perform the prover work for PG/merge. To facilitate
the new noir-bb interface that Cody and I have designed, we need to be
able to perform these two operations separately. This PR sets the stage
for that by re-expressing the `accumulate()` method of AztecIvc in terms
of two new methods: `complete_kernel_circuit_logic()` and
`execute_accumulation_prover()`.

The bulk of the work in doing this was similarly splitting up the method
`goblin.merge()` which performs both recursive verification for the
previous merge proof and merge proving for the circuit being
accumulated. We now have individual methods `goblin.prove_merge()` and
`goblin.verify_merge()` (which have been integrated into AztecIvc only).
Merge proofs are added to a `merge_verification_queue` as they are
created, which occurs on every accumulation round. When we get to a
kernel we recursively verify each proof then empty the queue. On the
first kernel there is a single merge proof to verify and in subsequent
kernels there are two.

closes AztecProtocol/barretenberg#1063

barretenberg/cpp/src/barretenberg/aztec_ivc/aztec_ivc.cpp

@@ -7,7 +7,7 @@ namespace bb {
  * @details If this is the first circuit being accumulated, initialize the prover and verifier accumulators. Otherwise,
  * fold the instance for the provided circuit into the accumulator. When two fold proofs have been enqueued, two
  * recursive folding verifications are appended to the next circuit that is accumulated, which must be a kernel.
- * Similarly, if a merge proof exists, a recursive merge verifier is appended.
+ * Similarly, merge proofs are stored in a queue and recursively verified in kernels.
  *
  * @param circuit Circuit to be accumulated/folded
  * @param precomputed_vk Optional precomputed VK (otherwise will be computed herein)
@@ -16,42 +16,73 @@ void AztecIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<Verifica
 {
     circuit_count++; // increment the count of circuits processed into the IVC
 
-    // When there are two fold proofs present, append two recursive verifiers to the kernel
-    if (verification_queue.size() == 2) {
-        BB_OP_COUNT_TIME_NAME("construct_circuits");
-        ASSERT(circuit_count % 2 == 0); // ensure this is a kernel
-
-        for (auto& [proof, vkey] : verification_queue) {
-            // Perform folding recursive verification
-            FoldingRecursiveVerifier verifier{ &circuit, { verifier_accumulator, { vkey } } };
-            auto verifier_accum = verifier.verify_folding_proof(proof);
-            verifier_accumulator = std::make_shared<VerifierInstance>(verifier_accum->get_value());
-
-            // Perform databus commitment consistency checks and propagate return data commitments via the public inputs
-            bus_depot.execute(verifier.instances);
-        }
-        verification_queue.clear();
+    // The aztec architecture dictates that every second circuit is a kernel. This check can be triggered/replaced by
+    // the presence of the recursive folding verify opcode once it is introduced into noir.
+    is_kernel = (circuit_count % 2 == 0);
+
+    // If present circuit is a kernel, perform required recursive PG and/or merge verifications and databus checks
+    if (is_kernel) {
+        complete_kernel_circuit_logic(circuit);
+    }
+
+    // Perform PG and/or merge proving
+    execute_accumulation_prover(circuit, precomputed_vk);
+}
+
+/**
+ * @brief Append logic to complete a kernel circuit
+ * @details A kernel circuit may contain some combination of PG recursive verification, merge recursive verification,
+ * and databus commitment consistency checks. This method appends this logic to a provided kernel circuit.
+ *
+ * @param circuit
+ */
+void AztecIVC::complete_kernel_circuit_logic(ClientCircuit& circuit)
+{
+    BB_OP_COUNT_TIME_NAME("construct_circuits");
+
+    // The folding verification queue should be either empty or contain two fold proofs
+    ASSERT(verification_queue.empty() || verification_queue.size() == 2);
+
+    for (auto& [proof, vkey] : verification_queue) {
+        // Perform folding recursive verification
+        FoldingRecursiveVerifier verifier{ &circuit, { verifier_accumulator, { vkey } } };
+        auto verifier_accum = verifier.verify_folding_proof(proof);
+        verifier_accumulator = std::make_shared<VerifierInstance>(verifier_accum->get_value());
+
+        // Perform databus commitment consistency checks and propagate return data commitments via public inputs
+        bus_depot.execute(verifier.instances);
     }
+    verification_queue.clear();
 
-    // Construct a merge proof (and add a recursive merge verifier to the circuit if a previous merge proof exists)
-    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1063): update recursive merge verification to only
-    // occur in kernels, similar to folding recursive verification.
-    goblin.merge(circuit);
+    // Recusively verify all merge proofs in queue
+    for (auto& proof : merge_verification_queue) {
+        goblin.verify_merge(circuit, proof);
+    }
+    merge_verification_queue.clear();
+}
+
+/**
+ * @brief Execute prover work for instance accumulation
+ * @details Construct an instance for the provided circuit. If this is the first instance in the IVC, simply initialize
+ * the folding accumulator. Otherwise, execute the PG prover to fold the instance into the accumulator and produce a
+ * folding proof. Also execute the merge protocol to produce a merge proof.
+ *
+ * @param circuit
+ * @param precomputed_vk
+ */
+void AztecIVC::execute_accumulation_prover(ClientCircuit& circuit,
+                                           const std::shared_ptr<VerificationKey>& precomputed_vk)
+{
+    // Construct merge proof for the present circuit and add to merge verification queue
+    MergeProof merge_proof = goblin.prove_merge(circuit);
+    merge_verification_queue.emplace_back(merge_proof);
 
     // Construct the prover instance for circuit
     auto prover_instance = std::make_shared<ProverInstance>(circuit, trace_structure);
 
     // Set the instance verification key from precomputed if available, else compute it
-    if (precomputed_vk) {
-        instance_vk = precomputed_vk;
-    } else {
-        instance_vk = std::make_shared<VerificationKey>(prover_instance->proving_key);
-    }
-
-    // Store whether the present circuit is a kernel (Note: the aztec architecture dictates that every second circuit
-    // is a kernel. This check can triggered/replaced by the presence of the recursive folding verify opcode once it is
-    // introduced into noir).
-    instance_vk->databus_propagation_data.is_kernel = (circuit_count % 2 == 0);
+    instance_vk = precomputed_vk ? precomputed_vk : std::make_shared<VerificationKey>(prover_instance->proving_key);
+    instance_vk->databus_propagation_data.is_kernel = is_kernel; // Store whether the present circuit is a kernel
 
     // If this is the first circuit simply initialize the prover and verifier accumulator instances
     if (circuit_count == 1) {
@@ -76,10 +107,12 @@ void AztecIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<Verifica
  */
 AztecIVC::Proof AztecIVC::prove()
 {
-    max_block_size_tracker.print();         // print minimum structured sizes for each block
-    ASSERT(verification_queue.size() == 1); // ensure only a single fold proof remains in the queue
-    auto& fold_proof = verification_queue[0].proof;
-    return { fold_proof, decider_prove(), goblin.prove() };
+    max_block_size_tracker.print();               // print minimum structured sizes for each block
+    ASSERT(verification_queue.size() == 1);       // ensure only a single fold proof remains in the queue
+    ASSERT(merge_verification_queue.size() == 1); // ensure only a single merge proof remains in the queue
+    FoldProof& fold_proof = verification_queue[0].proof;
+    MergeProof& merge_proof = merge_verification_queue[0];
+    return { fold_proof, decider_prove(), goblin.prove(merge_proof) };
 };
 
 bool AztecIVC::verify(const Proof& proof,

barretenberg/cpp/src/barretenberg/aztec_ivc/aztec_ivc.hpp

@@ -29,6 +29,7 @@ class AztecIVC {
     using VerificationKey = Flavor::VerificationKey;
     using FF = Flavor::FF;
     using FoldProof = std::vector<FF>;
+    using MergeProof = std::vector<FF>;
     using ProverInstance = ProverInstance_<Flavor>;
     using VerifierInstance = VerifierInstance_<Flavor>;
     using ClientCircuit = MegaCircuitBuilder; // can only be Mega
@@ -80,15 +81,24 @@ class AztecIVC {
 
     // Set of pairs of {fold_proof, verification_key} to be recursively verified
     std::vector<FoldingVerifierInputs> verification_queue;
+    // Set of merge proofs to be recursively verified
+    std::vector<MergeProof> merge_verification_queue;
 
     // Management of linking databus commitments between circuits in the IVC
     DataBusDepot bus_depot;
 
     // A flag indicating whether or not to construct a structured trace in the ProverInstance
     TraceStructure trace_structure = TraceStructure::NONE;
 
-    // The number of circuits processed into the IVC
-    size_t circuit_count = 0;
+    size_t circuit_count = 0; // the number of circuits processed into the IVC
+    bool is_kernel = false;   // is the present circuit a kernel
+
+    // Complete the logic of a kernel circuit (e.g. PG/merge recursive verification, databus consistency checks)
+    void complete_kernel_circuit_logic(ClientCircuit& circuit);
+
+    // Perform prover work for accumulation (e.g. PG folding, merge proving)
+    void execute_accumulation_prover(ClientCircuit& circuit,
+                                     const std::shared_ptr<VerificationKey>& precomputed_vk = nullptr);
 
     void accumulate(ClientCircuit& circuit, const std::shared_ptr<VerificationKey>& precomputed_vk = nullptr);
 

barretenberg/cpp/src/barretenberg/aztec_ivc/aztec_ivc.test.cpp

@@ -32,8 +32,10 @@ class AztecIVCTests : public ::testing::Test {
 
     /**
      * @brief Construct mock circuit with arithmetic gates and goblin ops
-     * @details Currently default sized to 2^16 to match kernel. (Note: dummy op gates added to avoid non-zero
-     * polynomials will bump size to next power of 2)
+     * @details Defaulted to add 2^16 gates (which will bump to next power of two with the addition of dummy gates).
+     * The size of the baseline circuit needs to be ~2x the number of gates appended to the kernel circuits via
+     * recursive verifications (currently ~60k) to ensure that the circuits being folded are equal in size. (This is
+     * only necessary if the structured trace is not in use).
      *
      */
     static Builder create_mock_circuit(AztecIVC& ivc, size_t log2_num_gates = 16)
@@ -88,10 +90,8 @@ TEST_F(AztecIVCTests, Basic)
 
 /**
  * @brief Check that the IVC fails to verify if an intermediate fold proof is invalid
- * @details When accumulating 4 circuits, there are 3 fold proofs to verify (the first two are recursively verfied
- and
- * the 3rd is verified as part of the IVC proof). Check that if any of one of these proofs is invalid, the IVC will
- fail
+ * @details When accumulating 4 circuits, there are 3 fold proofs to verify (the first two are recursively verfied and
+ * the 3rd is verified as part of the IVC proof). Check that if any of one of these proofs is invalid, the IVC will fail
  * to verify.
  *
  */
@@ -193,8 +193,6 @@ TEST_F(AztecIVCTests, BasicLarge)
         ivc.accumulate(circuit);
     }
 
-    info(ivc.goblin.op_queue->get_current_size());
-
     EXPECT_TRUE(ivc.prove_and_verify());
 };
 

barretenberg/cpp/src/barretenberg/goblin/goblin.hpp

@@ -40,16 +40,18 @@ class GoblinProver {
     using TranslatorProver = bb::TranslatorProver;
     using TranslatorProvingKey = bb::TranslatorFlavor::ProvingKey;
     using RecursiveMergeVerifier = bb::stdlib::recursion::goblin::MergeRecursiveVerifier_<MegaCircuitBuilder>;
+    using PairingPoints = RecursiveMergeVerifier::PairingPoints;
     using MergeProver = bb::MergeProver_<MegaFlavor>;
     using VerificationKey = MegaFlavor::VerificationKey;
+    using MergeProof = std::vector<FF>;
     /**
      * @brief Output of goblin::accumulate; an Ultra proof and the corresponding verification key
      *
      */
 
     std::shared_ptr<OpQueue> op_queue = std::make_shared<OpQueue>();
 
-    HonkProof merge_proof;
+    MergeProof merge_proof;
     GoblinProof goblin_proof;
 
     // on the first call to accumulate there is no merge proof to verify
@@ -115,13 +117,36 @@ class GoblinProver {
      */
     void merge(MegaCircuitBuilder& circuit_builder)
     {
-        BB_OP_COUNT_TIME_NAME("Goblin::merge");
-        // Complete the circuit logic by recursively verifying previous merge proof if it exists
+        // Append a recursive merge verification of the merge proof
         if (merge_proof_exists) {
-            RecursiveMergeVerifier merge_verifier{ &circuit_builder };
-            [[maybe_unused]] auto pairing_points = merge_verifier.verify_proof(merge_proof);
+            [[maybe_unused]] auto pairing_points = verify_merge(circuit_builder, merge_proof);
         }
 
+        // Construct a merge proof for the present circuit
+        merge_proof = prove_merge(circuit_builder);
+    };
+
+    /**
+     * @brief Append recursive verification of a merge proof to a provided circuit
+     *
+     * @param circuit_builder
+     * @return PairingPoints
+     */
+    PairingPoints verify_merge(MegaCircuitBuilder& circuit_builder, MergeProof& proof) const
+    {
+        BB_OP_COUNT_TIME_NAME("Goblin::merge");
+        RecursiveMergeVerifier merge_verifier{ &circuit_builder };
+        return merge_verifier.verify_proof(proof);
+    };
+
+    /**
+     * @brief Construct a merge proof for the goblin ECC ops in the provided circuit
+     *
+     * @param circuit_builder
+     */
+    MergeProof prove_merge(MegaCircuitBuilder& circuit_builder)
+    {
+        BB_OP_COUNT_TIME_NAME("Goblin::merge");
         // TODO(https://github.com/AztecProtocol/barretenberg/issues/993): Some circuits (particularly on the first call
         // to accumulate) may not have any goblin ecc ops prior to the call to merge(), so the commitment to the new
         // contribution (C_t_shift) in the merge prover will be the point at infinity. (Note: Some dummy ops are added
@@ -131,13 +156,12 @@ class GoblinProver {
             MockCircuits::construct_goblin_ecc_op_circuit(circuit_builder); // Add some arbitrary goblin ECC ops
         }
 
-        // Construct and store the merge proof to be recursively verified on the next call to accumulate
-        MergeProver merge_prover{ circuit_builder.op_queue };
-        merge_proof = merge_prover.construct_proof();
-
         if (!merge_proof_exists) {
             merge_proof_exists = true;
         }
+
+        MergeProver merge_prover{ circuit_builder.op_queue };
+        return merge_prover.construct_proof();
     };
 
     /**
@@ -171,9 +195,9 @@ class GoblinProver {
      *
      * @return Proof
      */
-    GoblinProof prove()
+    GoblinProof prove(MergeProof merge_proof_in = {})
     {
-        goblin_proof.merge_proof = std::move(merge_proof);
+        goblin_proof.merge_proof = merge_proof_in.empty() ? std::move(merge_proof) : std::move(merge_proof_in);
         prove_eccvm();
         prove_translator();
         return goblin_proof;

barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp

@@ -56,7 +56,7 @@ template <typename FF_> class MegaArith {
             this->lookup = FIXED_SIZE;
             this->busread = FIXED_SIZE;
             this->poseidon_external = FIXED_SIZE;
-            this->poseidon_internal = FIXED_SIZE;
+            this->poseidon_internal = 1 << 15;
         }
     };