feat: goblin op queue transcript aggregation (#2257)

Adds an ECC op queue aggregation protocol to the Goblin proof system.
I.e. given a previous aggregate transcript $T_{i-1}$ and the
contribution $t_i$ from the circuit being proven, establishes that $T_i
= T_{i-1} + X^{M_{i-1}}t_i$, where $M_{i-1}$ is the size of the
aggregate transcript at stage $i-1$. (Note: each $T_i$ actually
represents 4 polynomials, one for each column of the op queue. Similarly
for the others).

The protocol is encapsulated into a new prover "round" in between Gemini
and Shplonk. The polynomials $T_i, T_{i-1}, t_i^{shift}$ are committed
to as univariates, and their univariate opening proofs are appended to
the set of claims output from Gemini to be batched together in Shplonk.
The verifier confirms the relationship $T_i = T_{i-1} + X^{M_{i-1}}t_i$
via a Shwartz-Zippel check at a random challenge. This logic has been
added to both the native and recursive verifiers (`UltraVerifier_` and
`UltraRecursiveVerifier_`).

Some loose ends at this work does not address:
- Currently we commit to both $t_i$ and its shift. Established "tricks"
for avoiding this could work but the efficiency is not clear
- The aggregation protocol only works if there is a non-empty "previous"
transcript to aggregate. We get around this by mocking previous data for
now
- It should be possible to batch the transcript commitments using an
additional challenge (similar to what we do for the individual list
polynomials in the lookup argument) rather than send them all
individually. Left for future work.

Other notable changes in this PR:
- All Goblin functionality has been split out of the
`UltraCircuitBuilder` and moved into the new `GoblinUltraCircuitBuilder`
(defined via inheritance from the former). This facilitates idiomatic
differentiation of the two arithmetizations in the stdlib and also
isolates all goblin builder functionality in one place. (This was also
necessary to avoid breaking many tests that were hard coded with
assumptions about the number of constant variables added in the builder
composer).
- The recursive verifier tests have been fleshed out to cover the four
test cases that arise from combining a verifier algorithm (Ultra or
Goblin Ultra) with an arithmetization (Ultra or Goblin Ultra). To
facilitate this, the Recursive flavors are now templated by a
`BuilderType` which specifies the arithmetization of the circuit. For
example, we instantiate `UltraRecuirsiveVerifier` with
`flavor::GoblinUltra<UltraCircuitBuilder>` to obtain a circuit that
verifies a Goblin Ultra proof with a conventional Ultra arithmetization.
- Removes the hacky `use_goblin` flag from recursive verifier classes.
This is now achieved via a `IsGoblinBuilder<>` check now that Goblin has
its own builder.
- Constrains the op code values in the op wires via copy constraints
with constant variables (as suggested by Kesha)
- Adds a test of the "full" Goblin protocol (currently just Goblin Ultra
+ ECCVM). The Translator can be incorporated once it is complete

barretenberg/cpp/src/barretenberg/honk/composer/goblin/full_goblin_composer.test.cpp

@@ -0,0 +1,197 @@
+#include <cstddef>
+#include <cstdint>
+#include <gtest/gtest.h>
+
+#include "barretenberg/common/log.hpp"
+#include "barretenberg/honk/composer/eccvm_composer.hpp"
+#include "barretenberg/honk/composer/ultra_composer.hpp"
+#include "barretenberg/honk/proof_system/ultra_prover.hpp"
+#include "barretenberg/proof_system/circuit_builder/eccvm/eccvm_circuit_builder.hpp"
+#include "barretenberg/proof_system/circuit_builder/goblin_ultra_circuit_builder.hpp"
+#include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
+
+namespace test_full_goblin_composer {
+
+namespace {
+auto& engine = numeric::random::get_debug_engine();
+}
+
+class FullGoblinComposerTests : public ::testing::Test {
+  protected:
+    static void SetUpTestSuite()
+    {
+        barretenberg::srs::init_crs_factory("../srs_db/ignition");
+        barretenberg::srs::init_grumpkin_crs_factory("../srs_db/grumpkin");
+    }
+
+    using Curve = curve::BN254;
+    using FF = Curve::ScalarField;
+    using Point = Curve::AffineElement;
+    using CommitmentKey = proof_system::honk::pcs::CommitmentKey<Curve>;
+    using GoblinUltraBuilder = proof_system::GoblinUltraCircuitBuilder;
+    using GoblinUltraComposer = proof_system::honk::GoblinUltraComposer;
+    using ECCVMFlavor = proof_system::honk::flavor::ECCVMGrumpkin;
+    using ECCVMBuilder = proof_system::ECCVMCircuitBuilder<ECCVMFlavor>;
+    using ECCVMComposer = proof_system::honk::ECCVMComposer_<ECCVMFlavor>;
+    using VMOp = proof_system_eccvm::VMOperation<ECCVMFlavor::CycleGroup>;
+    static constexpr size_t NUM_OP_QUEUE_COLUMNS = proof_system::honk::flavor::GoblinUltra::NUM_WIRES;
+
+    /**
+     * @brief Generate a simple test circuit with some ECC op gates and conventional arithmetic gates
+     *
+     * @param builder
+     */
+    void generate_test_circuit(auto& builder)
+    {
+        // Add some arbitrary ecc op gates
+        for (size_t i = 0; i < 3; ++i) {
+            auto point = Point::random_element();
+            auto scalar = FF::random_element();
+            builder.queue_ecc_add_accum(point);
+            builder.queue_ecc_mul_accum(point, scalar);
+        }
+        builder.queue_ecc_eq();
+
+        // Add some conventional gates that utilize public inputs
+        for (size_t i = 0; i < 10; ++i) {
+            FF a = FF::random_element();
+            FF b = FF::random_element();
+            FF c = FF::random_element();
+            FF d = a + b + c;
+            uint32_t a_idx = builder.add_public_variable(a);
+            uint32_t b_idx = builder.add_variable(b);
+            uint32_t c_idx = builder.add_variable(c);
+            uint32_t d_idx = builder.add_variable(d);
+
+            builder.create_big_add_gate({ a_idx, b_idx, c_idx, d_idx, FF(1), FF(1), FF(1), FF(-1), FF(0) });
+        }
+    }
+
+    /**
+     * @brief Mock the interactions of a simple curcuit with the op_queue
+     * @details The transcript aggregation protocol in the Goblin proof system can not yet support an empty "previous
+     * transcript" (see issue #723). This function mocks the interactions with the op queue of a fictional "first"
+     * circuit. This way, when we go to generate a proof over our first "real" circuit, the transcript aggregation
+     * protocol can proceed nominally. The mock data is valid in the sense that it can be processed by all stages of
+     * Goblin as if it came from a genuine circuit.
+     *
+     * @param op_queue
+     */
+    static void perform_op_queue_interactions_for_mock_first_circuit(
+        std::shared_ptr<proof_system::ECCOpQueue>& op_queue)
+    {
+        auto builder = GoblinUltraBuilder(op_queue);
+
+        // Add a mul accum op and an equality op
+        auto point = Point::one() * FF::random_element();
+        auto scalar = FF::random_element();
+        builder.queue_ecc_mul_accum(point, scalar);
+        builder.queue_ecc_eq();
+
+        op_queue->set_size_data();
+
+        // Manually compute the op queue transcript commitments (which would normally be done by the prover)
+        auto crs_factory_ = barretenberg::srs::get_crs_factory();
+        auto commitment_key = CommitmentKey(op_queue->get_current_size(), crs_factory_);
+        std::array<Point, NUM_OP_QUEUE_COLUMNS> op_queue_commitments;
+        size_t idx = 0;
+        for (auto& entry : op_queue->get_aggregate_transcript()) {
+            op_queue_commitments[idx++] = commitment_key.commit(entry);
+        }
+        // Store the commitment data for use by the prover of the next circuit
+        op_queue->set_commitment_data(op_queue_commitments);
+    }
+};
+
+/**
+ * @brief Test proof construction/verification for a circuit with ECC op gates, public inputs, and basic arithmetic
+ * gates
+ * @note We simulate op queue interactions with a previous circuit so the actual circuit under test utilizes an op queue
+ * with non-empty 'previous' data. This avoid complications with zero-commitments etc.
+ *
+ */
+TEST_F(FullGoblinComposerTests, SimpleCircuit)
+{
+    auto op_queue = std::make_shared<proof_system::ECCOpQueue>();
+
+    // Add mock data to op queue to simulate interaction with a "first" circuit
+    perform_op_queue_interactions_for_mock_first_circuit(op_queue);
+
+    // Construct a series of simple Goblin circuits; generate and verify their proofs
+    size_t NUM_CIRCUITS = 3;
+    for (size_t circuit_idx = 0; circuit_idx < NUM_CIRCUITS; ++circuit_idx) {
+        auto builder = GoblinUltraBuilder(op_queue);
+
+        generate_test_circuit(builder);
+
+        auto composer = GoblinUltraComposer();
+        auto instance = composer.create_instance(builder);
+        auto prover = composer.create_prover(instance);
+        auto verifier = composer.create_verifier(instance);
+        auto proof = prover.construct_proof();
+        bool verified = verifier.verify_proof(proof);
+        EXPECT_EQ(verified, true);
+    }
+
+    // Construct an ECCVM circuit then generate and verify its proof
+    {
+        // Instantiate an ECCVM builder with the vm ops stored in the op queue
+        auto builder = ECCVMBuilder(op_queue->raw_ops);
+
+        // // Can fiddle with one of the operands to trigger a failure
+        // builder.vm_operations[0].z1 *= 2;
+
+        auto composer = ECCVMComposer();
+        auto prover = composer.create_prover(builder);
+        auto proof = prover.construct_proof();
+        auto verifier = composer.create_verifier(builder);
+        bool verified = verifier.verify_proof(proof);
+        ASSERT_TRUE(verified);
+    }
+}
+
+/**
+ * @brief Check that ECCVM verification fails if ECC op queue operands are tampered with
+ *
+ */
+TEST_F(FullGoblinComposerTests, SimpleCircuitFailureCase)
+{
+    auto op_queue = std::make_shared<proof_system::ECCOpQueue>();
+
+    // Add mock data to op queue to simulate interaction with a "first" circuit
+    perform_op_queue_interactions_for_mock_first_circuit(op_queue);
+
+    // Construct a series of simple Goblin circuits; generate and verify their proofs
+    size_t NUM_CIRCUITS = 3;
+    for (size_t circuit_idx = 0; circuit_idx < NUM_CIRCUITS; ++circuit_idx) {
+        auto builder = GoblinUltraBuilder(op_queue);
+
+        generate_test_circuit(builder);
+
+        auto composer = GoblinUltraComposer();
+        auto instance = composer.create_instance(builder);
+        auto prover = composer.create_prover(instance);
+        auto verifier = composer.create_verifier(instance);
+        auto proof = prover.construct_proof();
+        bool verified = verifier.verify_proof(proof);
+        EXPECT_EQ(verified, true);
+    }
+
+    // Construct an ECCVM circuit then generate and verify its proof
+    {
+        // Instantiate an ECCVM builder with the vm ops stored in the op queue
+        auto builder = ECCVMBuilder(op_queue->raw_ops);
+
+        // Fiddle with one of the operands to trigger a failure
+        builder.vm_operations[0].z1 += 1;
+
+        auto composer = ECCVMComposer();
+        auto prover = composer.create_prover(builder);
+        auto proof = prover.construct_proof();
+        auto verifier = composer.create_verifier(builder);
+        bool verified = verifier.verify_proof(proof);
+        EXPECT_EQ(verified, false);
+    }
+}
+
+} // namespace test_full_goblin_composer

barretenberg/cpp/src/barretenberg/honk/composer/goblin_ultra_composer.test.cpp

@@ -5,6 +5,7 @@
 #include "barretenberg/common/log.hpp"
 #include "barretenberg/honk/composer/ultra_composer.hpp"
 #include "barretenberg/honk/proof_system/ultra_prover.hpp"
+#include "barretenberg/proof_system/circuit_builder/goblin_ultra_circuit_builder.hpp"
 #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
 
 using namespace proof_system::honk;
@@ -18,51 +19,114 @@ auto& engine = numeric::random::get_debug_engine();
 class GoblinUltraHonkComposerTests : public ::testing::Test {
   protected:
     static void SetUpTestSuite() { barretenberg::srs::init_crs_factory("../srs_db/ignition"); }
+
+    using Curve = curve::BN254;
+    using FF = Curve::ScalarField;
+    using Point = Curve::AffineElement;
+    using CommitmentKey = pcs::CommitmentKey<Curve>;
+
+    /**
+     * @brief Generate a simple test circuit with some ECC op gates and conventional arithmetic gates
+     *
+     * @param builder
+     */
+    void generate_test_circuit(auto& builder)
+    {
+        // Add some ecc op gates
+        for (size_t i = 0; i < 3; ++i) {
+            auto point = Point::one() * FF::random_element();
+            auto scalar = FF::random_element();
+            builder.queue_ecc_mul_accum(point, scalar);
+        }
+        builder.queue_ecc_eq();
+
+        // Add some conventional gates that utilize public inputs
+        for (size_t i = 0; i < 10; ++i) {
+            FF a = FF::random_element();
+            FF b = FF::random_element();
+            FF c = FF::random_element();
+            FF d = a + b + c;
+            uint32_t a_idx = builder.add_public_variable(a);
+            uint32_t b_idx = builder.add_variable(b);
+            uint32_t c_idx = builder.add_variable(c);
+            uint32_t d_idx = builder.add_variable(d);
+
+            builder.create_big_add_gate({ a_idx, b_idx, c_idx, d_idx, FF(1), FF(1), FF(1), FF(-1), FF(0) });
+        }
+    }
+
+    /**
+     * @brief Construct a goblin ultra circuit then generate a verify its proof
+     *
+     * @param op_queue
+     * @return auto
+     */
+    bool construct_test_circuit_then_generate_and_verify_proof(auto& op_queue)
+    {
+        auto builder = proof_system::GoblinUltraCircuitBuilder(op_queue);
+
+        generate_test_circuit(builder);
+
+        auto composer = GoblinUltraComposer();
+        auto instance = composer.create_instance(builder);
+        auto prover = composer.create_prover(instance);
+        auto verifier = composer.create_verifier(instance);
+        auto proof = prover.construct_proof();
+        bool verified = verifier.verify_proof(proof);
+
+        return verified;
+    }
 };
 
 /**
  * @brief Test proof construction/verification for a circuit with ECC op gates, public inputs, and basic arithmetic
  * gates
+ * @note We simulate op queue interactions with a previous circuit so the actual circuit under test utilizes an op queue
+ * with non-empty 'previous' data. This avoid complications with zero-commitments etc.
  *
  */
-TEST_F(GoblinUltraHonkComposerTests, SimpleCircuit)
+TEST_F(GoblinUltraHonkComposerTests, SingleCircuit)
 {
-    using fr = barretenberg::fr;
-    using g1 = barretenberg::g1;
-    auto builder = proof_system::UltraCircuitBuilder();
-
-    // Define an arbitrary number of operations/gates
-    size_t num_ecc_ops = 3;
-    size_t num_conventional_gates = 10;
-
-    // Add some ecc op gates
-    for (size_t i = 0; i < num_ecc_ops; ++i) {
-        auto point = g1::affine_one * fr::random_element();
-        auto scalar = fr::random_element();
-        builder.queue_ecc_mul_accum(point, scalar);
-    }
+    auto op_queue = std::make_shared<proof_system::ECCOpQueue>();
 
-    // Add some conventional gates that utlize public inputs
-    for (size_t i = 0; i < num_conventional_gates; ++i) {
-        fr a = fr::random_element();
-        fr b = fr::random_element();
-        fr c = fr::random_element();
-        fr d = a + b + c;
-        uint32_t a_idx = builder.add_public_variable(a);
-        uint32_t b_idx = builder.add_variable(b);
-        uint32_t c_idx = builder.add_variable(c);
-        uint32_t d_idx = builder.add_variable(d);
-
-        builder.create_big_add_gate({ a_idx, b_idx, c_idx, d_idx, fr(1), fr(1), fr(1), fr(-1), fr(0) });
-    }
+    // Add mock data to op queue to simulate interaction with a previous circuit
+    op_queue->populate_with_mock_initital_data();
+
+    // Construct a test circuit then generate and verify its proof
+    auto verified = construct_test_circuit_then_generate_and_verify_proof(op_queue);
 
-    auto composer = GoblinUltraComposer();
-    auto instance = composer.create_instance(builder);
-    auto prover = composer.create_prover(instance);
-    auto verifier = composer.create_verifier(instance);
-    auto proof = prover.construct_proof();
-    bool verified = verifier.verify_proof(proof);
     EXPECT_EQ(verified, true);
 }
 
+/**
+ * @brief Test proof construction/verification for a circuit with ECC op gates, public inputs, and basic arithmetic
+ * gates
+ *
+ */
+TEST_F(GoblinUltraHonkComposerTests, MultipleCircuits)
+{
+    // Instantiate EccOpQueue. This will be shared across all circuits in the series
+    auto op_queue = std::make_shared<proof_system::ECCOpQueue>();
+
+    // Add mock data to op queue to simulate interaction with a previous circuit
+    op_queue->populate_with_mock_initital_data();
+
+    // Construct multiple test circuits that share an ECC op queue. Generate and verify a proof for each.
+    size_t NUM_CIRCUITS = 3;
+    for (size_t i = 0; i < NUM_CIRCUITS; ++i) {
+        construct_test_circuit_then_generate_and_verify_proof(op_queue);
+    }
+
+    // Compute the commitments to the aggregate op queue directly and check that they match those that were computed
+    // iteratively during transcript aggregation by the provers and stored in the op queue.
+    size_t aggregate_op_queue_size = op_queue->current_ultra_ops_size;
+    auto crs_factory = std::make_shared<barretenberg::srs::factories::FileCrsFactory<Curve>>("../srs_db/ignition");
+    auto commitment_key = std::make_shared<CommitmentKey>(aggregate_op_queue_size, crs_factory);
+    size_t idx = 0;
+    for (auto& result : op_queue->ultra_ops_commitments) {
+        auto expected = commitment_key->commit(op_queue->ultra_ops[idx++]);
+        EXPECT_EQ(result, expected);
+    }
+}
+
 } // namespace test_ultra_honk_composer

barretenberg/cpp/src/barretenberg/honk/flavor/ecc_vm.hpp

@@ -816,7 +816,7 @@ template <typename CycleGroup_T, typename Curve_T, typename PCS_T> class ECCVMBa
     };
 };
 
-class ECCVM : public ECCVMBase<grumpkin::g1, curve::BN254, pcs::kzg::KZG<curve::BN254, true>> {};
+class ECCVM : public ECCVMBase<grumpkin::g1, curve::BN254, pcs::kzg::KZG<curve::BN254>> {};
 class ECCVMGrumpkin : public ECCVMBase<barretenberg::g1, curve::Grumpkin, pcs::ipa::IPA<curve::Grumpkin>> {};
 
 // NOLINTEND(cppcoreguidelines-avoid-const-or-ref-data-members)

barretenberg/cpp/src/barretenberg/honk/flavor/goblin_ultra.hpp

@@ -2,7 +2,7 @@
 #include "barretenberg/honk/pcs/kzg/kzg.hpp"
 #include "barretenberg/honk/transcript/transcript.hpp"
 #include "barretenberg/polynomials/univariate.hpp"
-#include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
+#include "barretenberg/proof_system/circuit_builder/goblin_ultra_circuit_builder.hpp"
 #include "barretenberg/proof_system/flavor/flavor.hpp"
 #include "barretenberg/proof_system/relations/auxiliary_relation.hpp"
 #include "barretenberg/proof_system/relations/ecc_op_queue_relation.hpp"
@@ -16,13 +16,13 @@ namespace proof_system::honk::flavor {
 
 class GoblinUltra {
   public:
-    using CircuitBuilder = UltraCircuitBuilder;
+    using CircuitBuilder = GoblinUltraCircuitBuilder;
     using Curve = curve::BN254;
-    using PCS = pcs::kzg::KZG<Curve>;
+    using FF = Curve::ScalarField;
     using GroupElement = Curve::Element;
     using Commitment = Curve::AffineElement;
     using CommitmentHandle = Curve::AffineElement;
-    using FF = Curve::ScalarField;
+    using PCS = pcs::kzg::KZG<Curve>;
     using Polynomial = barretenberg::Polynomial<FF>;
     using PolynomialHandle = std::span<FF>;
     using CommitmentKey = pcs::CommitmentKey<Curve>;
@@ -288,6 +288,8 @@ class GoblinUltra {
 
         size_t num_ecc_op_gates; // needed to determine public input offset
 
+        std::shared_ptr<ECCOpQueue> op_queue;
+
         // The plookup wires that store plookup read data.
         std::array<PolynomialHandle, 3> get_table_column_wires() { return { w_l, w_r, w_o }; };
     };