feat: adding structure to Transcript (#2937)

Adding structuring feature to transcript based on the flavor and
resolves AztecProtocol/barretenberg#656.

The main two functions added are `deserialize_full_transcript()` and
`serialize_full_transcript()`, which are defined for each flavored
transcript derived class. These classes are defined in the flavor
classes, and primarily declare all of the transcript objects as member
variables.

`deserialize_full_transcript()` will take a full proof generated by the
flavored prover, and put all those bytes into the member variables.
Then, one can now view and modify the objects by just using the member
variables. In order to actually push the modification, one has to call
`serialize_full_transcript()` in order to regenerate the proof from the
(modified) member variables. The tests in `(flavor)_transcript.test.cpp`
illustrate how to use these functions in more depth. I also modified the
tests in `stdlib/recursion/honk/verifier/goblin_verifier.test.cpp` and
in `stdlib/recursion/honk/verifier/verifier.test.cpp`.

Got rid of the ProverTranscript and VerifierTranscript and merging them
into the BaseTranscript. This was done to avoid having to create these
for every flavor derived class and because they did not provide much on
top of the BaseTranscript.

Created transcript tests for flavors besides Ultra.

Transcript breakdown for all flavors:
https://docs.google.com/spreadsheets/d/1TpkkPj9o5ZXr_kWT7WsquXSEZY10C1Lt3BWeCA7Qe9U/edit#gid=0.

Some design considerations:
https://hackmd.io/o8HbTiujTGa4sLccAHQaxw?both.

barretenberg/cpp/src/barretenberg/honk/flavor/goblin_translator.hpp

@@ -1618,10 +1618,10 @@ template <size_t mini_circuit_size> class GoblinTranslator_ {
 
     class VerifierCommitments : public AllEntities<Commitment, CommitmentHandle> {
       public:
-        VerifierCommitments(std::shared_ptr<VerificationKey> verification_key, VerifierTranscript<FF> transcript)
+        VerifierCommitments(std::shared_ptr<VerificationKey> verification_key,
+                            [[maybe_unused]] const BaseTranscript<FF>& transcript)
         {
             static_cast<void>(transcript);
-            static_cast<void>(verification_key);
             this->lagrange_first = verification_key->lagrange_first;
             this->lagrange_last = verification_key->lagrange_last;
             this->lagrange_odd_in_minicircuit = verification_key->lagrange_odd_in_minicircuit;

barretenberg/cpp/src/barretenberg/honk/flavor/goblin_ultra.hpp

@@ -417,7 +417,8 @@ class GoblinUltra {
 
     class VerifierCommitments : public AllEntities<Commitment, CommitmentHandle> {
       public:
-        VerifierCommitments(std::shared_ptr<VerificationKey> verification_key, VerifierTranscript<FF> transcript)
+        VerifierCommitments(std::shared_ptr<VerificationKey> verification_key,
+                            [[maybe_unused]] const BaseTranscript<FF>& transcript)
         {
             static_cast<void>(transcript);
             q_m = verification_key->q_m;
@@ -454,6 +455,111 @@ class GoblinUltra {
         std::vector<FF> gate_separation_challenges;
         FF target_sum;
     };
+
+    /**
+     * @brief Derived class that defines proof structure for GoblinUltra proofs, as well as supporting functions.
+     *
+     */
+    class Transcript : public BaseTranscript<FF> {
+      public:
+        uint32_t circuit_size;
+        uint32_t public_input_size;
+        uint32_t pub_inputs_offset;
+        std::vector<FF> public_inputs;
+        Commitment w_l_comm;
+        Commitment w_r_comm;
+        Commitment w_o_comm;
+        Commitment ecc_op_wire_1_comm;
+        Commitment ecc_op_wire_2_comm;
+        Commitment ecc_op_wire_3_comm;
+        Commitment ecc_op_wire_4_comm;
+        Commitment sorted_accum_comm;
+        Commitment w_4_comm;
+        Commitment z_perm_comm;
+        Commitment z_lookup_comm;
+        std::vector<barretenberg::Univariate<FF, BATCHED_RELATION_PARTIAL_LENGTH>> sumcheck_univariates;
+        std::array<FF, NUM_ALL_ENTITIES> sumcheck_evaluations;
+        std::vector<Commitment> zm_cq_comms;
+        Commitment zm_cq_comm;
+        Commitment zm_pi_comm;
+
+        Transcript() = default;
+
+        Transcript(const std::vector<uint8_t>& proof)
+            : BaseTranscript<FF>(proof)
+        {}
+        void deserialize_full_transcript() override
+        {
+            // take current proof and put them into the struct
+            size_t num_bytes_read = 0;
+            circuit_size = deserialize_from_buffer<uint32_t>(proof_data, num_bytes_read);
+            size_t log_n = numeric::get_msb(circuit_size);
+
+            public_input_size = deserialize_from_buffer<uint32_t>(proof_data, num_bytes_read);
+            pub_inputs_offset = deserialize_from_buffer<uint32_t>(proof_data, num_bytes_read);
+            for (size_t i = 0; i < public_input_size; ++i) {
+                public_inputs.push_back(deserialize_from_buffer<FF>(proof_data, num_bytes_read));
+            }
+            w_l_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            w_r_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            w_o_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            ecc_op_wire_1_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            ecc_op_wire_2_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            ecc_op_wire_3_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            ecc_op_wire_4_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            sorted_accum_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            w_4_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            z_perm_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            z_lookup_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            for (size_t i = 0; i < log_n; ++i) {
+                sumcheck_univariates.push_back(
+                    deserialize_from_buffer<barretenberg::Univariate<FF, BATCHED_RELATION_PARTIAL_LENGTH>>(
+                        proof_data, num_bytes_read));
+            }
+            sumcheck_evaluations =
+                deserialize_from_buffer<std::array<FF, NUM_ALL_ENTITIES>>(proof_data, num_bytes_read);
+            for (size_t i = 0; i < log_n; ++i) {
+                zm_cq_comms.push_back(deserialize_from_buffer<Commitment>(proof_data, num_bytes_read));
+            }
+            zm_cq_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+            zm_pi_comm = deserialize_from_buffer<Commitment>(proof_data, num_bytes_read);
+        }
+
+        void serialize_full_transcript() override
+        {
+            size_t old_proof_length = proof_data.size();
+            proof_data.clear();
+            size_t log_n = numeric::get_msb(circuit_size);
+            serialize_to_buffer(circuit_size, proof_data);
+            serialize_to_buffer(public_input_size, proof_data);
+            serialize_to_buffer(pub_inputs_offset, proof_data);
+            for (size_t i = 0; i < public_input_size; ++i) {
+                serialize_to_buffer(public_inputs[i], proof_data);
+            }
+            serialize_to_buffer(w_l_comm, proof_data);
+            serialize_to_buffer(w_r_comm, proof_data);
+            serialize_to_buffer(w_o_comm, proof_data);
+            serialize_to_buffer(ecc_op_wire_1_comm, proof_data);
+            serialize_to_buffer(ecc_op_wire_2_comm, proof_data);
+            serialize_to_buffer(ecc_op_wire_3_comm, proof_data);
+            serialize_to_buffer(ecc_op_wire_4_comm, proof_data);
+            serialize_to_buffer(sorted_accum_comm, proof_data);
+            serialize_to_buffer(w_4_comm, proof_data);
+            serialize_to_buffer(z_perm_comm, proof_data);
+            serialize_to_buffer(z_lookup_comm, proof_data);
+            for (size_t i = 0; i < log_n; ++i) {
+                serialize_to_buffer(sumcheck_univariates[i], proof_data);
+            }
+            serialize_to_buffer(sumcheck_evaluations, proof_data);
+            for (size_t i = 0; i < log_n; ++i) {
+                serialize_to_buffer(zm_cq_comms[i], proof_data);
+            }
+            serialize_to_buffer(zm_cq_comm, proof_data);
+            serialize_to_buffer(zm_pi_comm, proof_data);
+
+            ASSERT(proof_data.size() == old_proof_length);
+        }
+    };
 };
 
 } // namespace proof_system::honk::flavor

barretenberg/cpp/src/barretenberg/honk/flavor/goblin_ultra_recursive.hpp

@@ -438,6 +438,141 @@ template <typename BuilderType> class GoblinUltraRecursive_ {
             this->lagrange_ecc_op = verification_key->lagrange_ecc_op;
         }
     };
+
+    /**
+     * @brief Derived class that defines proof structure for GoblinUltraRecursive proofs, as well as supporting
+     * functions.
+     *
+     */
+    class Transcript : public BaseTranscript<FF> {
+      public:
+        // Transcript objects defined as public member variables for easy access and modification
+        uint32_t circuit_size;
+        uint32_t public_input_size;
+        uint32_t pub_inputs_offset;
+        std::vector<FF> public_inputs;
+        Commitment w_l_comm;
+        Commitment w_r_comm;
+        Commitment w_o_comm;
+        Commitment ecc_op_wire_1_comm;
+        Commitment ecc_op_wire_2_comm;
+        Commitment ecc_op_wire_3_comm;
+        Commitment ecc_op_wire_4_comm;
+        Commitment sorted_accum_comm;
+        Commitment w_4_comm;
+        Commitment z_perm_comm;
+        Commitment z_lookup_comm;
+        std::vector<barretenberg::Univariate<FF, BATCHED_RELATION_PARTIAL_LENGTH>> sumcheck_univariates;
+        std::array<FF, NUM_ALL_ENTITIES> sumcheck_evaluations;
+        std::vector<Commitment> zm_cq_comms;
+        Commitment zm_cq_comm;
+        Commitment zm_pi_comm;
+
+        Transcript() = default;
+
+        // Used by verifier to initialize the transcript
+        Transcript(const std::vector<uint8_t>& proof)
+            : BaseTranscript<FF>(proof)
+        {}
+
+        static Transcript prover_init_empty()
+        {
+            Transcript transcript;
+            constexpr uint32_t init{ 42 }; // arbitrary
+            transcript.send_to_verifier("Init", init);
+            return transcript;
+        };
+
+        static Transcript verifier_init_empty(const Transcript& transcript)
+        {
+            Transcript verifier_transcript{ transcript.proof_data };
+            [[maybe_unused]] auto _ = verifier_transcript.template receive_from_prover<uint32_t>("Init");
+            return verifier_transcript;
+        };
+
+        /**
+         * @brief Takes a FULL GoblinUltraRecursive proof and deserializes it into the public member variables that
+         * compose the structure. Must be called in order to access the structure of the proof.
+         *
+         */
+        void deserialize_full_transcript() override
+        {
+            // take current proof and put them into the struct
+            size_t num_bytes_read = 0;
+            circuit_size = deserialize_from_buffer<uint32_t>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            size_t log_n = numeric::get_msb(circuit_size);
+
+            public_input_size = deserialize_from_buffer<uint32_t>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            pub_inputs_offset = deserialize_from_buffer<uint32_t>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            for (size_t i = 0; i < public_input_size; ++i) {
+                public_inputs.push_back(deserialize_from_buffer<FF>(BaseTranscript<FF>::proof_data, num_bytes_read));
+            }
+            w_l_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            w_r_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            w_o_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            ecc_op_wire_1_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            ecc_op_wire_2_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            ecc_op_wire_3_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            ecc_op_wire_4_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            sorted_accum_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            w_4_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            z_perm_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            z_lookup_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            for (size_t i = 0; i < log_n; ++i) {
+                sumcheck_univariates.push_back(
+                    deserialize_from_buffer<barretenberg::Univariate<FF, BATCHED_RELATION_PARTIAL_LENGTH>>(
+                        BaseTranscript<FF>::proof_data, num_bytes_read));
+            }
+            sumcheck_evaluations = deserialize_from_buffer<std::array<FF, NUM_ALL_ENTITIES>>(
+                BaseTranscript<FF>::proof_data, num_bytes_read);
+            for (size_t i = 0; i < log_n; ++i) {
+                zm_cq_comms.push_back(
+                    deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read));
+            }
+            zm_cq_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+            zm_pi_comm = deserialize_from_buffer<Commitment>(BaseTranscript<FF>::proof_data, num_bytes_read);
+        }
+        /**
+         * @brief Serializes the structure variables into a FULL GoblinUltraRecursive proof. Should be called only if
+         * deserialize_full_transcript() was called and some transcript variable was modified.
+         *
+         */
+        void serialize_full_transcript() override
+        {
+            size_t old_proof_length = BaseTranscript<FF>::proof_data.size();
+            BaseTranscript<FF>::proof_data.clear(); // clear proof_data so the rest of the function can replace it
+            size_t log_n = numeric::get_msb(circuit_size);
+            serialize_to_buffer(circuit_size, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(public_input_size, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(pub_inputs_offset, BaseTranscript<FF>::proof_data);
+            for (size_t i = 0; i < public_input_size; ++i) {
+                serialize_to_buffer(public_inputs[i], BaseTranscript<FF>::proof_data);
+            }
+            serialize_to_buffer(w_l_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(w_r_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(w_o_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(ecc_op_wire_1_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(ecc_op_wire_2_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(ecc_op_wire_3_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(ecc_op_wire_4_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(sorted_accum_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(w_4_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(z_perm_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(z_lookup_comm, BaseTranscript<FF>::proof_data);
+            for (size_t i = 0; i < log_n; ++i) {
+                serialize_to_buffer(sumcheck_univariates[i], BaseTranscript<FF>::proof_data);
+            }
+            serialize_to_buffer(sumcheck_evaluations, BaseTranscript<FF>::proof_data);
+            for (size_t i = 0; i < log_n; ++i) {
+                serialize_to_buffer(zm_cq_comms[i], BaseTranscript<FF>::proof_data);
+            }
+            serialize_to_buffer(zm_cq_comm, BaseTranscript<FF>::proof_data);
+            serialize_to_buffer(zm_pi_comm, BaseTranscript<FF>::proof_data);
+
+            // sanity check to make sure we generate the same length of proof as before.
+            ASSERT(BaseTranscript<FF>::proof_data.size() == old_proof_length);
+        }
+    };
 };
 
 } // namespace proof_system::honk::flavor