Lde/lookup grand prod relation (AztecProtocol/barretenberg#359)

* lookup grand product relation tests passing

* ignore final entry in table polys for consistency chec, same reason as for selectors

* adding eta and lookup gp delta to relation params

* incorporate lookup gp init relation into tests

* correcting the degree of lookup relation

circuits/cpp/barretenberg/cpp/src/barretenberg/honk/composer/composer_helper/ultra_honk_composer_helper.cpp

@@ -270,15 +270,18 @@ std::shared_ptr<plonk::proving_key> UltraHonkComposerHelper<CircuitConstructor>:
     // // all four columns. We don't want to have equal commitments, because biggroup operations assume no points are
     // // equal, so if we tried to verify an ultra proof in a circuit, the biggroup operations would fail. To combat
     // // this, we just choose distinct values:
-    size_t num_selectors = circuit_constructor.num_selectors;
-    ASSERT(offset == subgroup_size - 1);
-    auto unique_last_value = num_selectors + 1; // Note: in compute_proving_key_base, moments earlier, each selector
-                                                // vector was given a unique last value from 1..num_selectors. So we
-                                                // avoid those values and continue the count, to ensure uniqueness.
-    poly_q_table_column_1[subgroup_size - 1] = unique_last_value;
-    poly_q_table_column_2[subgroup_size - 1] = ++unique_last_value;
-    poly_q_table_column_3[subgroup_size - 1] = ++unique_last_value;
-    poly_q_table_column_4[subgroup_size - 1] = ++unique_last_value;
+
+    // TODO(#217)(luke): Similar to the selectors, enforcing non-zero values by inserting an arbitrary final element
+    // in the table polys will result in lookup relations not being satisfied. Address this with issue #217.
+    // size_t num_selectors = circuit_constructor.num_selectors;
+    // ASSERT(offset == subgroup_size - 1);
+    // auto unique_last_value = num_selectors + 1; // Note: in compute_proving_key_base, moments earlier, each selector
+    //                                             // vector was given a unique last value from 1..num_selectors. So we
+    //                                             // avoid those values and continue the count, to ensure uniqueness.
+    // poly_q_table_column_1[subgroup_size - 1] = unique_last_value;
+    // poly_q_table_column_2[subgroup_size - 1] = ++unique_last_value;
+    // poly_q_table_column_3[subgroup_size - 1] = ++unique_last_value;
+    // poly_q_table_column_4[subgroup_size - 1] = ++unique_last_value;
 
     circuit_proving_key->polynomial_store.put("table_value_1_lagrange", std::move(poly_q_table_column_1));
     circuit_proving_key->polynomial_store.put("table_value_2_lagrange", std::move(poly_q_table_column_2));

circuits/cpp/barretenberg/cpp/src/barretenberg/honk/composer/standard_honk_composer.test.cpp

@@ -7,7 +7,7 @@
 #include "barretenberg/honk/sumcheck/sumcheck_round.hpp"
 #include "barretenberg/honk/sumcheck/relations/grand_product_computation_relation.hpp"
 #include "barretenberg/honk/sumcheck/relations/grand_product_initialization_relation.hpp"
-#include "barretenberg/honk/utils/public_inputs.hpp"
+#include "barretenberg/honk/utils/grand_product_delta.hpp"
 
 #include <gtest/gtest.h>
 

circuits/cpp/barretenberg/cpp/src/barretenberg/honk/composer/ultra_honk_composer.test.cpp

@@ -10,7 +10,7 @@
 #include "barretenberg/honk/sumcheck/sumcheck_round.hpp"
 #include "barretenberg/honk/sumcheck/relations/grand_product_computation_relation.hpp"
 #include "barretenberg/honk/sumcheck/relations/grand_product_initialization_relation.hpp"
-#include "barretenberg/honk/utils/public_inputs.hpp"
+#include "barretenberg/honk/utils/grand_product_delta.hpp"
 
 // TODO(luke): TEMPORARY; for testing only (comparison with Ultra Plonk composers)
 #include "barretenberg/plonk/composer/ultra_composer.hpp"
@@ -39,34 +39,30 @@ std::vector<uint32_t> add_variables(auto& composer, std::vector<fr> variables)
  * @param honk_prover
  * @param plonk_prover
  */
-// NOTE: Currently checking exact consistency for witness polynomials (wires, sorted lists) and table polys.
-// The permutation polys are computed differently between plonk and honk so we do not expect consistency.
-// Equality is checked on all selectors but we ignore the final entry since we do not enforce non-zero selectors in
-// Honk.
 void verify_consistency(honk::UltraProver& honk_prover, plonk::UltraProver& plonk_prover)
 {
     auto& honk_store = honk_prover.key->polynomial_store;
     auto& plonk_store = plonk_prover.key->polynomial_store;
 
-    // Check that all selectors agree (aside from the final element which will differ due to not enforcing non-zero
-    // selectors in Honk).
+    // Check that all selectors and table polynomials agree (aside from the final element which will differ
+    // due to not enforcing non-zero polynomials in Honk).
     for (auto& entry : honk_store) {
         std::string key = entry.first;
         bool is_selector = (key.find("q_") != std::string::npos) || (key.find("table_type") != std::string::npos);
-        if (plonk_store.contains(key) && is_selector) {
+        bool is_table = (key.find("table_value_") != std::string::npos);
+        if (plonk_store.contains(key) && (is_selector || is_table)) {
             // check equality for all but final entry
             for (size_t i = 0; i < honk_store.get(key).size() - 1; ++i) {
                 ASSERT_EQ(honk_store.get(key)[i], plonk_store.get(key)[i]);
             }
         }
     }
 
-    // Check that sorted witness-table and table polys agree
+    // Check that sorted witness-table polynomials agree
     for (auto& entry : honk_store) {
         std::string key = entry.first;
         bool is_sorted_table = (key.find("s_") != std::string::npos);
-        bool is_table = (key.find("table_value_") != std::string::npos);
-        if (plonk_store.contains(key) && (is_sorted_table || is_table)) {
+        if (plonk_store.contains(key) && is_sorted_table) {
             ASSERT_EQ(honk_store.get(key), plonk_store.get(key));
         }
     }

circuits/cpp/barretenberg/cpp/src/barretenberg/honk/flavor/flavor.hpp

@@ -219,6 +219,10 @@ struct UltraArithmetization {
         ID_2,
         ID_3,
         ID_4,
+        TABLE_1,
+        TABLE_2,
+        TABLE_3,
+        TABLE_4,
         LAGRANGE_FIRST,
         LAGRANGE_LAST, // = LAGRANGE_N-1 whithout ZK, but can be less
         /* --- WITNESS POLYNOMIALS --- */
@@ -230,11 +234,19 @@ struct UltraArithmetization {
         S_2,
         S_3,
         S_4,
+        S_ACCUM,
         Z_PERM,
         Z_LOOKUP,
         /* --- SHIFTED POLYNOMIALS --- */
         W_1_SHIFT,
+        W_2_SHIFT,
+        W_3_SHIFT,
         W_4_SHIFT,
+        TABLE_1_SHIFT,
+        TABLE_2_SHIFT,
+        TABLE_3_SHIFT,
+        TABLE_4_SHIFT,
+        S_ACCUM_SHIFT,
         Z_PERM_SHIFT,
         Z_LOOKUP_SHIFT,
         /* --- --- */

circuits/cpp/barretenberg/cpp/src/barretenberg/honk/sumcheck/relations/grand_product_computation_relation.hpp

@@ -2,6 +2,7 @@
 #include "relation.hpp"
 #include "barretenberg/honk/flavor/flavor.hpp"
 #include "../polynomials/univariate.hpp"
+// TODO(luke): change name of this file to permutation_grand_product_relation(s).hpp and move 'init' relation into it.
 
 namespace proof_system::honk::sumcheck {
 

circuits/cpp/barretenberg/cpp/src/barretenberg/honk/sumcheck/relations/lookup_grand_product_relation.hpp

@@ -0,0 +1,205 @@
+#pragma once
+#include "relation.hpp"
+#include "barretenberg/honk/flavor/flavor.hpp"
+#include "../polynomials/univariate.hpp"
+
+namespace proof_system::honk::sumcheck {
+
+template <typename FF> class LookupGrandProductComputationRelation {
+  public:
+    // 1 + polynomial degree of this relation
+    static constexpr size_t RELATION_LENGTH = 6; // deg(z_lookup * column_selector * wire * q_lookup * table) = 5
+    using MULTIVARIATE = proof_system::honk::UltraArithmetization::POLYNOMIAL;
+
+    /**
+     * @brief Compute contribution of the lookup grand prod relation for a given edge (internal function)
+     *
+     * @details This the relation confirms faithful calculation of the lookup grand
+     * product polynomial Z_lookup. The contribution is
+     *      z_lookup * (1 + β) * [q_lookup * f + γ] * (t_accum_k + βt_accum_{k+1} + γ(1 + β)) -
+     *      z_lookup_shift * (s_accum_k + βs_accum_{k+1} + γ(1 + β))
+     * where
+     *      f = (w_1 + q_2*w_1_shift) + η(w_2 + q_m*w_2_shift) + η²(w_3 + q_c*w_3_shift) + η³q_index,
+     *      t_accum = table_1 + ηtable_2 + η²table_3 + η³table_4, and
+     *      s_accum = s_1 + ηs_2 + η²s_3 + η³s_4.
+     * Note: Selectors q_2, q_m and q_c are repurposed as 'column step size' for lookup gates.
+     *
+     * @param evals transformed to `evals + C(extended_edges(X)...)*scaling_factor`
+     * @param extended_edges an std::array containing the fully extended Univariate edges.
+     * @param parameters contains beta, gamma, and public_input_delta, ....
+     * @param scaling_factor optional term to scale the evaluation before adding to evals.
+     */
+    inline void add_edge_contribution(Univariate<FF, RELATION_LENGTH>& evals,
+                                      const auto& extended_edges,
+                                      const RelationParameters<FF>& relation_parameters,
+                                      const FF& scaling_factor) const
+    {
+        const auto& eta = relation_parameters.eta;
+        const auto& beta = relation_parameters.beta;
+        const auto& gamma = relation_parameters.gamma;
+        const auto& grand_product_delta = relation_parameters.lookup_grand_product_delta;
+
+        const auto one_plus_beta = FF::one() + beta;
+        const auto gamma_by_one_plus_beta = gamma * one_plus_beta;
+        const auto eta_sqr = eta * eta;
+        const auto eta_cube = eta_sqr * eta;
+
+        auto w_1 = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::W_L]);
+        auto w_2 = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::W_R]);
+        auto w_3 = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::W_O]);
+
+        auto w_1_shift = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::W_1_SHIFT]);
+        auto w_2_shift = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::W_2_SHIFT]);
+        auto w_3_shift = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::W_3_SHIFT]);
+
+        auto table_1 = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::TABLE_1]);
+        auto table_2 = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::TABLE_2]);
+        auto table_3 = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::TABLE_3]);
+        auto table_4 = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::TABLE_4]);
+
+        auto table_1_shift = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::TABLE_1_SHIFT]);
+        auto table_2_shift = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::TABLE_2_SHIFT]);
+        auto table_3_shift = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::TABLE_3_SHIFT]);
+        auto table_4_shift = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::TABLE_4_SHIFT]);
+
+        auto s_accum = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::S_ACCUM]);
+        auto s_accum_shift = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::S_ACCUM_SHIFT]);
+
+        auto z_lookup = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::Z_LOOKUP]);
+        auto z_lookup_shift = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::Z_LOOKUP_SHIFT]);
+
+        auto table_index = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::Q_O]);
+        auto column_1_step_size = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::Q_R]);
+        auto column_2_step_size = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::Q_M]);
+        auto column_3_step_size = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::Q_C]);
+        auto q_lookup = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::QLOOKUPTYPE]);
+
+        auto lagrange_first = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::LAGRANGE_FIRST]);
+        auto lagrange_last = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::LAGRANGE_LAST]);
+
+        // (w_1 + q_2*w_1_shift) + η(w_2 + q_m*w_2_shift) + η²(w_3 + q_c*w_3_shift) + η³q_index.
+        auto wire_accum = (w_1 + column_1_step_size * w_1_shift) + (w_2 + column_2_step_size * w_2_shift) * eta +
+                          (w_3 + column_3_step_size * w_3_shift) * eta_sqr + table_index * eta_cube;
+
+        // t_1 + ηt_2 + η²t_3 + η³t_4
+        auto table_accum = table_1 + table_2 * eta + table_3 * eta_sqr + table_4 * eta_cube;
+        // t_1_shift + ηt_2_shift + η²t_3_shift + η³t_4_shift
+        auto table_accum_shift =
+            table_1_shift + table_2_shift * eta + table_3_shift * eta_sqr + table_4_shift * eta_cube;
+
+        // Contribution (1)
+        auto tmp = (q_lookup * wire_accum + gamma);
+        tmp *= (table_accum + table_accum_shift * beta + gamma_by_one_plus_beta);
+        tmp *= one_plus_beta;
+        tmp *= (z_lookup + lagrange_first);
+        tmp -= (z_lookup_shift + lagrange_last * grand_product_delta) *
+               (s_accum + s_accum_shift * beta + gamma_by_one_plus_beta);
+        evals += tmp * scaling_factor;
+    };
+
+    void add_full_relation_value_contribution(FF& full_honk_relation_value,
+                                              auto& purported_evaluations,
+                                              const RelationParameters<FF>& relation_parameters) const
+    {
+        const auto& eta = relation_parameters.eta;
+        const auto& beta = relation_parameters.beta;
+        const auto& gamma = relation_parameters.gamma;
+        const auto& grand_product_delta = relation_parameters.lookup_grand_product_delta;
+
+        const auto one_plus_beta = FF::one() + beta;
+        const auto gamma_by_one_plus_beta = gamma * one_plus_beta;
+        const auto eta_sqr = eta * eta;
+        const auto eta_cube = eta_sqr * eta;
+
+        auto w_1 = purported_evaluations[MULTIVARIATE::W_L];
+        auto w_2 = purported_evaluations[MULTIVARIATE::W_R];
+        auto w_3 = purported_evaluations[MULTIVARIATE::W_O];
+
+        auto w_1_shift = purported_evaluations[MULTIVARIATE::W_1_SHIFT];
+        auto w_2_shift = purported_evaluations[MULTIVARIATE::W_2_SHIFT];
+        auto w_3_shift = purported_evaluations[MULTIVARIATE::W_3_SHIFT];
+
+        auto table_1 = purported_evaluations[MULTIVARIATE::TABLE_1];
+        auto table_2 = purported_evaluations[MULTIVARIATE::TABLE_2];
+        auto table_3 = purported_evaluations[MULTIVARIATE::TABLE_3];
+        auto table_4 = purported_evaluations[MULTIVARIATE::TABLE_4];
+
+        auto table_1_shift = purported_evaluations[MULTIVARIATE::TABLE_1_SHIFT];
+        auto table_2_shift = purported_evaluations[MULTIVARIATE::TABLE_2_SHIFT];
+        auto table_3_shift = purported_evaluations[MULTIVARIATE::TABLE_3_SHIFT];
+        auto table_4_shift = purported_evaluations[MULTIVARIATE::TABLE_4_SHIFT];
+
+        auto s_accum = purported_evaluations[MULTIVARIATE::S_ACCUM];
+        auto s_accum_shift = purported_evaluations[MULTIVARIATE::S_ACCUM_SHIFT];
+        auto z_lookup = purported_evaluations[MULTIVARIATE::Z_LOOKUP];
+        auto z_lookup_shift = purported_evaluations[MULTIVARIATE::Z_LOOKUP_SHIFT];
+
+        auto table_index = purported_evaluations[MULTIVARIATE::Q_O];
+        auto column_1_step_size = purported_evaluations[MULTIVARIATE::Q_R];
+        auto column_2_step_size = purported_evaluations[MULTIVARIATE::Q_M];
+        auto column_3_step_size = purported_evaluations[MULTIVARIATE::Q_C];
+        auto q_lookup = purported_evaluations[MULTIVARIATE::QLOOKUPTYPE];
+
+        auto lagrange_first = purported_evaluations[MULTIVARIATE::LAGRANGE_FIRST];
+        auto lagrange_last = purported_evaluations[MULTIVARIATE::LAGRANGE_LAST];
+
+        // (w_1 + q_2*w_1_shift) + η(w_2 + q_m*w_2_shift) + η²(w_3 + q_c*w_3_shift) + η³q_index.
+        auto wire_accum = (w_1 + column_1_step_size * w_1_shift) + (w_2 + column_2_step_size * w_2_shift) * eta +
+                          (w_3 + column_3_step_size * w_3_shift) * eta_sqr + table_index * eta_cube;
+
+        // t_1 + ηt_2 + η²t_3 + η³t_4
+        auto table_accum = table_1 + table_2 * eta + table_3 * eta_sqr + table_4 * eta_cube;
+        // t_1_shift + ηt_2_shift + η²t_3_shift + η³t_4_shift
+        auto table_accum_shift =
+            table_1_shift + table_2_shift * eta + table_3_shift * eta_sqr + table_4_shift * eta_cube;
+
+        // Contribution (1)
+        auto tmp = (q_lookup * wire_accum + gamma);
+        tmp *= (table_accum + beta * table_accum_shift + gamma_by_one_plus_beta);
+        tmp *= one_plus_beta;
+        tmp *= (z_lookup + lagrange_first);
+        tmp -= (z_lookup_shift + lagrange_last * grand_product_delta) *
+               (s_accum + beta * s_accum_shift + gamma_by_one_plus_beta);
+        full_honk_relation_value += tmp;
+    };
+};
+
+template <typename FF> class LookupGrandProductInitializationRelation {
+  public:
+    // 1 + polynomial degree of this relation
+    static constexpr size_t RELATION_LENGTH = 3; // deg(lagrange_last * z_lookup_shift) = 2
+    using MULTIVARIATE = proof_system::honk::UltraArithmetization::POLYNOMIAL;
+
+    /**
+     * @brief Compute contribution of the lookup grand prod relation for a given edge (internal function)
+     *
+     * @details This the relation confirms correct initialization of the lookup grand
+     * product polynomial Z_lookup with Z_lookup[circuit_size] = 0.
+     *
+     * @param evals transformed to `evals + C(extended_edges(X)...)*scaling_factor`
+     * @param extended_edges an std::array containing the fully extended Univariate edges.
+     * @param parameters contains beta, gamma, and public_input_delta, ....
+     * @param scaling_factor optional term to scale the evaluation before adding to evals.
+     */
+    inline void add_edge_contribution(Univariate<FF, RELATION_LENGTH>& evals,
+                                      const auto& extended_edges,
+                                      const RelationParameters<FF>& /*unused*/,
+                                      const FF& scaling_factor) const
+    {
+        auto z_lookup_shift = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::Z_LOOKUP_SHIFT]);
+        auto lagrange_last = UnivariateView<FF, RELATION_LENGTH>(extended_edges[MULTIVARIATE::LAGRANGE_LAST]);
+
+        evals += (lagrange_last * z_lookup_shift) * scaling_factor;
+    };
+
+    void add_full_relation_value_contribution(FF& full_honk_relation_value,
+                                              auto& purported_evaluations,
+                                              const RelationParameters<FF>& /*unused*/) const
+    {
+        auto z_lookup_shift = purported_evaluations[MULTIVARIATE::Z_LOOKUP_SHIFT];
+        auto lagrange_last = purported_evaluations[MULTIVARIATE::LAGRANGE_LAST];
+
+        full_honk_relation_value += lagrange_last * z_lookup_shift;
+    };
+};
+} // namespace proof_system::honk::sumcheck

circuits/cpp/barretenberg/cpp/src/barretenberg/honk/sumcheck/relations/relation.hpp

@@ -1,10 +1,18 @@
 #pragma once
 
+#include <cstddef>
 namespace proof_system::honk::sumcheck {
 
+/**
+ * @brief Container for parameters used by the grand product (permutation, lookup) Honk relations
+ *
+ * @tparam FF
+ */
 template <typename FF> struct RelationParameters {
-    FF beta;
-    FF gamma;
-    FF public_input_delta;
+    FF eta = FF::zero();                        // Lookup
+    FF beta = FF::zero();                       // Permutation + Lookup
+    FF gamma = FF::zero();                      // Permutation + Lookup
+    FF public_input_delta = FF::zero();         // Permutation
+    FF lookup_grand_product_delta = FF::zero(); // Lookup
 };
 } // namespace proof_system::honk::sumcheck