fix: Move timers for ClientIVC breakdown (#5145)

AztecProtocol/aztec-packages#4841 improved the witness construction in the client IVC benchmarks to make use of straightforward parallelism. This PR updates the manual % breakdown script to reflect the new structure. It also fixes benchmark.sh so that it can be used inside of benchmark_client_ivc.sh in the case where the benchmarking instance is unavailable. Result: ``` -------------------------------------------------------------------------------- Benchmark Time CPU Iterations UserCounters... -------------------------------------------------------------------------------- ClientIVCBench/Full/6 29212 ms 24574 ms 1 Decider::construct_proof=1 Decider::construct_proof(t)=755.662M ECCVMComposer::create_prover=1 ECCVMComposer::create_prover(t)=3.41214G ECCVMProver::construct_proof=1 ECCVMProver::construct_proof(t)=1.77115G Goblin::merge=11 Goblin::merge(t)=128.511M GoblinTranslatorCircuitBuilder::constructor=1 GoblinTranslatorCircuitBuilder::constructor(t)=56.2447M GoblinTranslatorComposer::create_prover=1 GoblinTranslatorComposer::create_prover(t)=121.099M GoblinTranslatorProver::construct_proof=1 GoblinTranslatorProver::construct_proof(t)=920.195M ProtogalaxyProver::fold_instances=10 ProtogalaxyProver::fold_instances(t)=15.8816G ProverInstance(Circuit&)=11 ProverInstance(Circuit&)(t)=1.94818G batch_mul_with_endomorphism=30 batch_mul_with_endomorphism(t)=564.471M commit=425 commit(t)=3.96399G compute_combiner=10 compute_combiner(t)=7.19205G compute_perturbator=9 compute_perturbator(t)=1.3842G construct_circuits=6 construct_circuits(t)=4.19883G Benchmarking lock deleted. client_ivc_bench.json 100% 2904 89.0KB/s 00:00 function ms % sum construct_circuits(t) 4199 14.41% ProverInstance(Circuit&)(t) 1948 6.69% ProtogalaxyProver::fold_instances(t) 15882 54.51% Decider::construct_proof(t) 756 2.59% ECCVMComposer::create_prover(t) 3412 11.71% GoblinTranslatorComposer::create_prover(t) 121 0.42% ECCVMProver::construct_proof(t) 1771 6.08% GoblinTranslatorProver::construct_proof(t) 920 3.16% Goblin::merge(t) 129 0.44% Total time accounted for: 29137ms/29212ms = 99.74% ```
AztecProtocol · Mar 13, 2024 · 3094154 · 3094154
1 parent cd12ce1
commit 3094154
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 33 deletions.
diff --git a/cpp/scripts/analyze_client_ivc_bench.py b/cpp/scripts/analyze_client_ivc_bench.py
@@ -7,8 +7,7 @@
 
 # Single out an independent set of functions accounting for most of BENCHMARK's real_time
 to_keep = [
-    "construct_mock_function_circuit(t)",
-    "construct_mock_folding_kernel(t)",
+    "construct_circuits(t)",
     "ProverInstance(Circuit&)(t)",
     "ProtogalaxyProver::fold_instances(t)",
     "Decider::construct_proof(t)",
@@ -42,3 +41,9 @@
 totals = totals.format(
     sum_of_kept_times_ms, total_time_ms, sum_of_kept_times_ms/total_time_ms)
 print(totals)
+
+print('\nBreakdown of ECCVMProver::create_prover:')
+for key in ["ECCVMComposer::compute_witness(t)", "ECCVMComposer::create_proving_key(t)"]:
+    time_ms = bench[key]/1e6
+    total_time_ms = bench["ECCVMComposer::create_prover(t)"]/1e6
+    print(f"{key:<{MAX_LABEL_LENGTH}}{time_ms:>8.0f}  {time_ms/total_time_ms:>8.2%}")
diff --git a/cpp/scripts/benchmark.sh b/cpp/scripts/benchmark.sh
@@ -2,16 +2,19 @@
 set -eu
 
 BENCHMARK=${1:-goblin_bench}
-COMMAND=${2:-./bin/$BENCHMARK}
+COMMAND=${2:-./$BENCHMARK}
+PRESET=${3:-clang16}
+BUILD_DIR=${4:-build}
+
 
 # Move above script dir.
 cd $(dirname $0)/..
 
 # Configure and build.
-cmake --preset clang16
-cmake --build --preset clang16 --target $BENCHMARK 
+cmake --preset $PRESET
+cmake --build --preset $PRESET --target $BENCHMARK 
 
-cd build
+cd $BUILD_DIR
 # Consistency with _wasm.sh targets / shorter $COMMAND.
 cp ./bin/$BENCHMARK .
 $COMMAND
diff --git a/cpp/scripts/benchmark_client_ivc.sh b/cpp/scripts/benchmark_client_ivc.sh
@@ -21,5 +21,5 @@ cd $BUILD_DIR
 scp $BB_SSH_KEY $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build/$TARGET.json .
 
 # Analyze the results
-cd $(dirname $0)/..
+cd ../
 python3 ./scripts/analyze_client_ivc_bench.py
diff --git a/cpp/src/barretenberg/benchmark/client_ivc_bench/client_ivc.bench.cpp b/cpp/src/barretenberg/benchmark/client_ivc_bench/client_ivc.bench.cpp
@@ -2,6 +2,7 @@
 #include <benchmark/benchmark.h>
 
 #include "barretenberg/client_ivc/client_ivc.hpp"
+#include "barretenberg/common/op_count.hpp"
 #include "barretenberg/common/op_count_google_bench.hpp"
 #include "barretenberg/goblin/mock_circuits.hpp"
 #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
@@ -46,9 +47,12 @@ class ClientIVCBench : public benchmark::Fixture {
         std::vector<Builder> initial_function_circuits(2);
 
         // Construct 2 starting function circuits in parallel
-        parallel_for(2, [&](size_t circuit_index) {
-            GoblinMockCircuits::construct_mock_function_circuit(initial_function_circuits[circuit_index]);
-        });
+        {
+            BB_OP_COUNT_TIME_NAME("construct_circuits");
+            parallel_for(2, [&](size_t circuit_index) {
+                GoblinMockCircuits::construct_mock_function_circuit(initial_function_circuits[circuit_index]);
+            });
+        };
 
         // Prepend queue to the first circuit
         initial_function_circuits[0].op_queue->prepend_previous_queue(*ivc.goblin.op_queue);
@@ -81,25 +85,28 @@ class ClientIVCBench : public benchmark::Fixture {
             Builder kernel_circuit{ size_hint, ivc.goblin.op_queue };
             Builder function_circuit{ size_hint };
             // Construct function and kernel circuits in parallel
-            parallel_for(2, [&](size_t workload_idx) {
-                // workload index is 0 for kernel and 1 for function
-                if (workload_idx == 0) {
-                    if (circuit_idx == 0) {
-
-                        // Create the first folding kernel which only verifies the accumulation of a
-                        // function circuit
-                        kernel_verifier_accumulator = GoblinMockCircuits::construct_mock_folding_kernel(
-                            kernel_circuit, function_fold_output, {}, kernel_verifier_accumulator);
+            {
+                BB_OP_COUNT_TIME_NAME("construct_circuits");
+                parallel_for(2, [&](size_t workload_idx) {
+                    // workload index is 0 for kernel and 1 for function
+                    if (workload_idx == 0) {
+                        if (circuit_idx == 0) {
+
+                            // Create the first folding kernel which only verifies the accumulation of a
+                            // function circuit
+                            kernel_verifier_accumulator = GoblinMockCircuits::construct_mock_folding_kernel(
+                                kernel_circuit, function_fold_output, {}, kernel_verifier_accumulator);
+                        } else {
+                            // Create kernel circuit containing the recursive folding verification of a function circuit
+                            // and a kernel circuit
+                            kernel_verifier_accumulator = GoblinMockCircuits::construct_mock_folding_kernel(
+                                kernel_circuit, function_fold_output, kernel_fold_output, kernel_verifier_accumulator);
+                        }
                     } else {
-                        // Create kernel circuit containing the recursive folding verification of a function circuit and
-                        // a kernel circuit
-                        kernel_verifier_accumulator = GoblinMockCircuits::construct_mock_folding_kernel(
-                            kernel_circuit, function_fold_output, kernel_fold_output, kernel_verifier_accumulator);
+                        GoblinMockCircuits::construct_mock_function_circuit(function_circuit);
                     }
-                } else {
-                    GoblinMockCircuits::construct_mock_function_circuit(function_circuit);
-                }
-            });
+                });
+            };
 
             // No need to prepend queue, it's the same after last swap
             // Accumulate kernel circuit
@@ -127,14 +134,20 @@ class ClientIVCBench : public benchmark::Fixture {
             // Create and accumulate the first folding kernel which only verifies the accumulation of a function circuit
             Builder kernel_circuit{ size_hint, ivc.goblin.op_queue };
             auto kernel_verifier_accumulator = std::make_shared<ClientIVC::VerifierInstance>(ivc.vks.first_func_vk);
-            kernel_verifier_accumulator = GoblinMockCircuits::construct_mock_folding_kernel(
-                kernel_circuit, function_fold_output, {}, kernel_verifier_accumulator);
+            {
+                BB_OP_COUNT_TIME_NAME("construct_circuits");
+                kernel_verifier_accumulator = GoblinMockCircuits::construct_mock_folding_kernel(
+                    kernel_circuit, function_fold_output, {}, kernel_verifier_accumulator);
+            }
             auto kernel_fold_proof = ivc.accumulate(kernel_circuit);
             kernel_fold_output = { kernel_fold_proof, ivc.vks.first_kernel_vk };
         } else {
             Builder kernel_circuit{ size_hint, ivc.goblin.op_queue };
-            kernel_verifier_accumulator = GoblinMockCircuits::construct_mock_folding_kernel(
-                kernel_circuit, function_fold_output, kernel_fold_output, kernel_verifier_accumulator);
+            {
+                BB_OP_COUNT_TIME_NAME("construct_circuits");
+                kernel_verifier_accumulator = GoblinMockCircuits::construct_mock_folding_kernel(
+                    kernel_circuit, function_fold_output, kernel_fold_output, kernel_verifier_accumulator);
+            }
 
             auto kernel_fold_proof = ivc.accumulate(kernel_circuit);
             kernel_fold_output = { kernel_fold_proof, ivc.vks.kernel_vk };

diff --git a/cpp/src/barretenberg/eccvm/eccvm_composer.cpp b/cpp/src/barretenberg/eccvm/eccvm_composer.cpp
@@ -10,6 +10,8 @@ namespace bb {
  */
 template <IsECCVMFlavor Flavor> void ECCVMComposer_<Flavor>::compute_witness(CircuitConstructor& circuit_constructor)
 {
+    BB_OP_COUNT_TIME_NAME("ECCVMComposer::compute_witness");
+
     if (computed_witness) {
         return;
     }
@@ -67,6 +69,8 @@ template <IsECCVMFlavor Flavor>
 std::shared_ptr<typename Flavor::ProvingKey> ECCVMComposer_<Flavor>::compute_proving_key(
     CircuitConstructor& circuit_constructor)
 {
+    BB_OP_COUNT_TIME_NAME("ECCVMComposer::create_proving_key");
+
     if (proving_key) {
         return proving_key;
     }

diff --git a/cpp/src/barretenberg/eccvm/eccvm_composer.hpp b/cpp/src/barretenberg/eccvm/eccvm_composer.hpp
@@ -70,6 +70,7 @@ template <IsECCVMFlavor Flavor> class ECCVMComposer_ {
 
     void compute_commitment_key(size_t circuit_size)
     {
+        BB_OP_COUNT_TIME_NAME("ECCVMComposer::compute_commitment_key");
         commitment_key = std::make_shared<CommitmentKey>(circuit_size);
     };
 };

diff --git a/cpp/src/barretenberg/goblin/mock_circuits.hpp b/cpp/src/barretenberg/goblin/mock_circuits.hpp
@@ -109,7 +109,6 @@ class GoblinMockCircuits {
      */
     static void construct_mock_function_circuit(GoblinUltraBuilder& builder, bool large = false)
     {
-        BB_OP_COUNT_TIME();
         // Determine number of times to execute the below operations that constitute the mock circuit logic. Note that
         // the circuit size does not scale linearly with number of iterations due to e.g. amortization of lookup costs
         const size_t NUM_ITERATIONS_LARGE = 13; // results in circuit size 2^19 (521327 gates)
@@ -233,7 +232,6 @@ class GoblinMockCircuits {
         const VerifierFoldData& kernel,
         std::shared_ptr<VerifierInstance>& prev_kernel_accum)
     {
-        BB_OP_COUNT_TIME();
         using GURecursiveFlavor = GoblinUltraRecursiveFlavor_<GoblinUltraBuilder>;
         using RecursiveVerifierInstances =
             bb::stdlib::recursion::honk::RecursiveVerifierInstances_<GURecursiveFlavor, 2>;