feat: Hook up secondary calldata column in dsl (#7759)

Previously we could use a single calldata and return_data from noir with
support from the bberg backend. With
[this](https://github.com/noir-lang/noir/pull/5599/files) PR, noir has
an interface for multiple calldata entities. The backend has support for
two calldata columns (`calldata` and `secondary_calldata`). This work
hooks up a second calldata column in dsl.

The main limitation of this work is that there is no way to distinguish
between the two calldata columns in dsl. This is OK for the operations
within a single circuit because in that context there is no important
distinction between the two calldata columns (`calldata`,
`secondary_calldata`). It does cause a problem however in the mechanism
for linking two circuits via the databus. This is because we need to
know which calldata corresponds to app data and which corresponds to
previous kernel data in order to prove that the connection was made
faithfully. The ideal solution is probably to treat `secondary_calldata`
(possibly rename to `app_calldata`?) as a unique entity in noir (similar
to how `calldata` and `return_data` are treated as different entities),
rather than allowing arbitrarily many individual `calldata` entities. I
made an issue
[here](AztecProtocol/barretenberg#1070).

---------

Co-authored-by: sirasistant <sirasistant@gmail.com>

barretenberg/acir_tests/reset_acir_tests.sh

@@ -1,8 +1,12 @@
 # Run from within barretenberg/acir_tests
+
+# clean and rebuild noir then compile the test programs
 cd ../../noir/noir-repo
 cargo clean
 noirup -p .
 cd test_programs && ./rebuild.sh
 
+# remove and repopulate the test artifacts in bberg
 cd ../../../barretenberg/acir_tests
 rm -rf acir_tests
+./clone_test_vectors.sh

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.cpp

@@ -190,6 +190,7 @@ void build_constraints(Builder& builder,
     }
 
     // Add block constraints
+    assign_calldata_ids<Builder>(constraint_system.block_constraints);
     for (size_t i = 0; i < constraint_system.block_constraints.size(); ++i) {
         const auto& constraint = constraint_system.block_constraints.at(i);
         create_block_constraints(builder, constraint, has_valid_witness_assignments);

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_integration.test.cpp

@@ -69,6 +69,7 @@ class AcirIntegrationTest : public ::testing::Test {
         info("log circuit size   = ", prover.instance->proving_key.log_circuit_size);
 #endif
         auto proof = prover.construct_proof();
+
         // Verify Honk proof
         auto verification_key = std::make_shared<VerificationKey>(prover.instance->proving_key);
         Verifier verifier{ verification_key };
@@ -430,7 +431,7 @@ INSTANTIATE_TEST_SUITE_P(AcirTests,
                          testing::Values("fold_basic", "fold_basic_nested_call"));
 
 /**
- *@brief A basic test of a circuit generated in noir that makes use of the databus
+ * @brief A basic test of a circuit generated in noir that makes use of the databus
  *
  */
 TEST_F(AcirIntegrationTest, DISABLED_Databus)
@@ -452,6 +453,59 @@ TEST_F(AcirIntegrationTest, DISABLED_Databus)
     EXPECT_TRUE(prove_and_verify_honk<Flavor>(builder));
 }
 
+/**
+ * @brief Test a program that uses two databus calldata columns
+ * @details In addition to checking that a proof of the resulting circuit verfies, check that the specific structure of
+ * the calldata/return data interaction in the noir program is reflected in the bberg circuit
+ */
+TEST_F(AcirIntegrationTest, DISABLED_DatabusTwoCalldata)
+{
+    using Flavor = MegaFlavor;
+    using Builder = Flavor::CircuitBuilder;
+
+    std::string test_name = "databus_two_calldata";
+    info("Test: ", test_name);
+    acir_format::AcirProgram acir_program = get_program_data_from_test_file(test_name);
+
+    // Construct a bberg circuit from the acir representation
+    Builder builder = acir_format::create_circuit<Builder>(acir_program.constraints, 0, acir_program.witness);
+
+    // Check that the databus columns in the builder have been populated as expected
+    const auto& calldata = builder.get_calldata();
+    const auto& secondary_calldata = builder.get_secondary_calldata();
+    const auto& return_data = builder.get_return_data();
+
+    ASSERT(calldata.size() == 4);
+    ASSERT(secondary_calldata.size() == 3);
+    ASSERT(return_data.size() == 4);
+
+    // Check that return data was computed from the two calldata inputs as expected
+    ASSERT_EQ(builder.get_variable(calldata[0]) + builder.get_variable(secondary_calldata[0]),
+              builder.get_variable(return_data[0]));
+    ASSERT_EQ(builder.get_variable(calldata[1]) + builder.get_variable(secondary_calldata[1]),
+              builder.get_variable(return_data[1]));
+    ASSERT_EQ(builder.get_variable(calldata[2]) + builder.get_variable(secondary_calldata[2]),
+              builder.get_variable(return_data[2]));
+    ASSERT_EQ(builder.get_variable(calldata[3]), builder.get_variable(return_data[3]));
+
+    // Ensure that every index of each bus column was read once as expected
+    for (size_t idx = 0; idx < calldata.size(); ++idx) {
+        ASSERT_EQ(calldata.get_read_count(idx), 1);
+    }
+    for (size_t idx = 0; idx < secondary_calldata.size(); ++idx) {
+        ASSERT_EQ(secondary_calldata.get_read_count(idx), 1);
+    }
+    for (size_t idx = 0; idx < return_data.size(); ++idx) {
+        ASSERT_EQ(return_data.get_read_count(idx), 1);
+    }
+
+    // This prints a summary of the types of gates in the circuit
+    builder.blocks.summarize();
+
+    // Construct and verify Honk proof
+    EXPECT_TRUE(prove_and_verify_honk<Flavor>(builder));
+}
+
 /**
  * @brief Ensure that adding gates post-facto to a circuit generated from acir still results in a valid circuit
  * @details This is a pattern required by e.g. ClientIvc which appends recursive verifiers to acir-generated circuits

barretenberg/cpp/src/barretenberg/dsl/acir_format/block_constraint.cpp

@@ -163,20 +163,34 @@ void process_call_data_operations(Builder& builder,
     using databus_ct = stdlib::databus<Builder>;
 
     databus_ct databus;
-    // Populate the calldata in the databus
-    databus.calldata.set_values(init);
-    for (const auto& op : constraint.trace) {
-        ASSERT(op.access_type == 0);
-        field_ct value = poly_to_field_ct(op.value, builder);
-        field_ct index = poly_to_field_ct(op.index, builder);
-        fr w_value = 0;
-        if (has_valid_witness_assignments) {
-            // If witness are assigned, we use the correct value for w
-            w_value = index.get_value();
+
+    // Method for processing operations on a generic databus calldata array
+    auto process_calldata = [&](auto& calldata_array) {
+        calldata_array.set_values(init); // Initialize the data in the bus array
+
+        for (const auto& op : constraint.trace) {
+            ASSERT(op.access_type == 0);
+            field_ct value = poly_to_field_ct(op.value, builder);
+            field_ct index = poly_to_field_ct(op.index, builder);
+            fr w_value = 0;
+            if (has_valid_witness_assignments) {
+                // If witness are assigned, we use the correct value for w
+                w_value = index.get_value();
+            }
+            field_ct w = field_ct::from_witness(&builder, w_value);
+            value.assert_equal(calldata_array[w]);
+            w.assert_equal(index);
         }
-        field_ct w = field_ct::from_witness(&builder, w_value);
-        value.assert_equal(databus.calldata[w]);
-        w.assert_equal(index);
+    };
+
+    // Process primary or secondary calldata based on calldata_id
+    if (constraint.calldata_id == 0) {
+        process_calldata(databus.calldata);
+    } else if (constraint.calldata_id == 1) {
+        process_calldata(databus.secondary_calldata);
+    } else {
+        info("Databus only supports two calldata arrays.");
+        ASSERT(false);
     }
 }
 
@@ -199,4 +213,20 @@ void process_return_data_operations(const BlockConstraint& constraint, std::vect
     ASSERT(constraint.trace.size() == 0);
 }
 
+// Do nothing for Ultra since it does not support Databus
+template <> void assign_calldata_ids<UltraCircuitBuilder>([[maybe_unused]] std::vector<BlockConstraint>& constraints) {}
+
+template <> void assign_calldata_ids<MegaCircuitBuilder>(std::vector<BlockConstraint>& constraints)
+{
+    // Assign unique ID to each calldata block constraint
+    uint32_t calldata_id = 0;
+    for (auto& constraint : constraints) {
+        if (constraint.type == BlockType::CallData) {
+            constraint.calldata_id = calldata_id++;
+        }
+    }
+    // The backend only supports 2 calldata columns
+    ASSERT(calldata_id <= 2);
+}
+
 } // namespace acir_format

barretenberg/cpp/src/barretenberg/dsl/acir_format/block_constraint.hpp

@@ -22,6 +22,7 @@ struct BlockConstraint {
     std::vector<bb::poly_triple> init;
     std::vector<MemOp> trace;
     BlockType type;
+    uint32_t calldata_id{ 0 };
 };
 
 template <typename Builder>
@@ -47,6 +48,16 @@ void process_call_data_operations(Builder& builder,
 template <typename Builder>
 void process_return_data_operations(const BlockConstraint& constraint, std::vector<bb::stdlib::field_t<Builder>>& init);
 
+/**
+ * @brief Assign a unique ID to each calldata block constraint based on the order in which it was recieved
+ * TODO(https://github.com/AztecProtocol/barretenberg/issues/1070): this is a workaround to allow calldata inputs to be
+ * distinguished by the backend since no identifiers are received from noir.
+ *
+ * @tparam Builder
+ * @param constraints
+ */
+template <typename Builder> void assign_calldata_ids(std::vector<BlockConstraint>& constraints);
+
 template <typename B> inline void read(B& buf, MemOp& mem_op)
 {
     using serialize::read;

noir/noir-repo/test_programs/execution_success/databus_two_calldata/Nargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "databus_two_calldata"
+type = "bin"
+authors = [""]
+
+[dependencies]

noir/noir-repo/test_programs/execution_success/databus_two_calldata/Prover.toml

@@ -0,0 +1,3 @@
+x = [0,1,2,3]
+y = [0,2,4]
+z = [1,3,5,7]

noir/noir-repo/test_programs/execution_success/databus_two_calldata/src/main.nr

@@ -0,0 +1,11 @@
+// An simple program demonstrating two calldata array inputs and a single return data array. As an arbitrary example, 
+// the return data is computed as a linear combination of the calldata.
+fn main(mut x: [u32; 4], y: call_data(0) [u32; 3], z: call_data(1) [u32; 4]) -> return_data [u32; 4] {
+    let mut result = [0; 4];
+    for i in 0..3 {
+        let idx = x[i];
+        result[idx] = y[idx] + z[idx];
+    }
+    result[x[3]] = z[x[3]];
+    result
+}

noir/noir-repo/test_programs/execution_success/databus_two_calldata_simple/Nargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "databus_two_calldata_simple"
+type = "bin"
+authors = [""]
+
+[dependencies]

noir/noir-repo/test_programs/execution_success/databus_two_calldata_simple/Prover.toml

@@ -0,0 +1,3 @@
+idx = "1"
+y = [7, 9]
+z = [1,2,3,4]

noir/noir-repo/test_programs/execution_success/databus_two_calldata_simple/src/main.nr

@@ -0,0 +1,5 @@
+fn main(mut idx: u32, y: call_data(0) [u32; 2], z: call_data(1) [u32; 4]) -> return_data u32 {
+    let a = y[idx];
+    let b = z[idx];
+    a + b
+}