Setup bench framework to run experiments

soroban-env-host/Cargo.toml

@@ -32,6 +32,9 @@ k256 = {version = "0.13.1", features=["ecdsa", "arithmetic"]}
 # is needed to build the host for wasm (a rare but supported config).
 getrandom = { version = "0.2", features=["js"] }
 sha3 = "0.10.8"
+# NB: this must match the same curve25519-dalek version used by ed25519-dalek above
+# used only for calibration
+curve25519-dalek = { version = "4", default-features = false, features = ["digest"]}
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
 tracy-client = { version = "=0.15.2", features = ["enable", "timer-fallback"], default-features = false, optional = true }

soroban-env-host/benches/common/experimental/ed25519_scalar_mul.rs

@@ -0,0 +1,62 @@
+use crate::common::HostCostMeasurement;
+use curve25519_dalek::{constants, edwards, scalar};
+use rand::{rngs::StdRng, Rng, RngCore};
+use soroban_env_host::{
+    cost_runner::{Ed25519ScalarMulRun, Ed25519ScalarMulSample},
+    Host,
+};
+
+// This measures the costs of doing an Ed25519 scalar multiply, which is a
+// component of signature verification and is the only part advertised as
+// variable time. The input value is ignored. The underlying operation is
+// nonlinear and the point of this measurement is to investigate how wide the
+// variation is in the histogram.
+#[allow(non_snake_case)]
+pub(crate) struct Ed25519ScalarMulMeasure {
+    a: scalar::Scalar,
+    A: edwards::EdwardsPoint,
+    b: scalar::Scalar,
+}
+#[allow(non_snake_case)]
+impl HostCostMeasurement for Ed25519ScalarMulMeasure {
+    type Runner = Ed25519ScalarMulRun;
+
+    fn new_best_case(_host: &Host, _rng: &mut StdRng) -> Ed25519ScalarMulSample {
+        let a = scalar::Scalar::ONE;
+        let b = scalar::Scalar::ONE;
+        let A = constants::ED25519_BASEPOINT_COMPRESSED
+            .decompress()
+            .unwrap();
+        Ed25519ScalarMulSample { a, b, A }
+    }
+
+    fn new_worst_case(_host: &Host, _rng: &mut StdRng, _input: u64) -> Ed25519ScalarMulSample {
+        let a = scalar::Scalar::from_bytes_mod_order([0xff; 32]);
+        let b = a.clone();
+        let A = constants::ED25519_BASEPOINT_COMPRESSED
+            .decompress()
+            .unwrap();
+        Ed25519ScalarMulSample { a, b, A }
+    }
+
+    fn new_random_case(_host: &Host, rng: &mut StdRng, _input: u64) -> Ed25519ScalarMulSample {
+        fn random_scalar(rng: &mut StdRng) -> scalar::Scalar {
+            let mut buf: [u8; 32] = [0; 32];
+            rng.fill_bytes(&mut buf);
+            // Here we zero-out some number of bits from the top, making the scalar
+            // smaller and thus the work less. It's not really plausible for natural
+            // occurrences here to have more than a handful of contiguous zeroes here
+            // but for thoroughness we'll let it go up to 64 (8 bytes).
+            for i in 0..rng.gen_range(0..9) {
+                buf[31 - i] = 0
+            }
+            scalar::Scalar::from_bytes_mod_order(buf)
+        }
+        let a = random_scalar(rng);
+        let b = random_scalar(rng);
+        let A = constants::ED25519_BASEPOINT_COMPRESSED
+            .decompress()
+            .unwrap();
+        Ed25519ScalarMulSample { a, b, A }
+    }
+}

soroban-env-host/benches/common/experimental/mod.rs

@@ -0,0 +1,3 @@
+mod ed25519_scalar_mul;
+
+pub(crate) use ed25519_scalar_mul::*;

soroban-env-host/benches/common/measure.rs

@@ -1,9 +1,8 @@
 use rand::{rngs::StdRng, Rng, SeedableRng};
 use soroban_bench_utils::{tracking_allocator::AllocationGroupToken, HostTracker};
-use soroban_env_common::xdr::ContractCostType;
 use soroban_env_host::{
     budget::{AsBudget, COST_MODEL_LIN_TERM_SCALE_BITS},
-    cost_runner::CostRunner,
+    cost_runner::{CostRunner, CostType},
     Host,
 };
 use std::{io, ops::Range};
@@ -32,7 +31,7 @@ impl Measurements {
     // as a basic spot check.
     fn check_one_baseline_range(
         &self,
-        cost: ContractCostType,
+        cost: &CostType,
         meas: &str,
         f: impl Fn(&Measurement) -> u64,
     ) -> Result<(), io::Error> {
@@ -49,7 +48,7 @@ impl Measurements {
     // Confirms that there's a reasonable range of values above the baseline;
     // only relevant for certain measurements, currently no way to tell
     // systematically, so gated behind env var
-    pub fn check_range_against_baseline(&self, cost: ContractCostType) -> Result<(), io::Error> {
+    pub fn check_range_against_baseline(&self, cost: &CostType) -> Result<(), io::Error> {
         if std::env::var("CHECK_RANGE_AGAINST_BASELINE").is_ok() {
             self.check_one_baseline_range(cost, "cpu", |m| m.cpu_insns)?;
             self.check_one_baseline_range(cost, "mem", |m| m.mem_bytes)?;
@@ -452,17 +451,29 @@ pub fn measure_worst_case_costs<HCM: HostCostMeasurement>(
     })
 }
 
+/// Measure the cost variation of a HCM. `sweep_input` specifies whether the input
+/// is fixed or randomized.
+///     if true - input is randomized, with `large_input` specifying the upperbound
+///               of the input size
+///     if false - input size is fixed at `large_input`
+/// `iteration` specifies number of iterations to run the measurement
+/// `include_best_case` specifies whether best case is included. Often the best case
+/// is a trivial case that isn't too relevant (and never hit). So if one is more
+/// interested in the worst/average analysis, it might be useful to throw it away.
 pub fn measure_cost_variation<HCM: HostCostMeasurement>(
     large_input: u64,
+    iterations: u64,
+    sweep_input: bool,
+    include_best_case: bool,
 ) -> Result<Measurements, std::io::Error> {
-    let count = 100;
     let mut i = 0;
     let mut rng = StdRng::from_seed([0xff; 32]);
 
     // run baseline measure
+    let mut base_range = std::iter::once(0);
     let baseline = measure_costs_inner::<HCM, _, _>(
         |host| {
-            std::iter::once(0)
+            base_range
                 .next()
                 .map(|_| HCM::new_baseline_case(host, &mut rng))
         },
@@ -476,13 +487,21 @@ pub fn measure_cost_variation<HCM: HostCostMeasurement>(
     let measurements = measure_costs_inner::<HCM, _, _>(
         |host| {
             i += 1;
+            let input = if sweep_input {
+                rng.gen_range(1..=2 + large_input)
+            } else {
+                large_input
+            };
             match i {
-                1 => Some(HCM::new_best_case(host, &mut rng)),
-                2 => Some(HCM::new_worst_case(host, &mut rng, large_input)),
-                n if n < count => {
-                    let input = rng.gen_range(1..=2 + large_input);
-                    Some(HCM::new_random_case(host, &mut rng, input))
+                1 => {
+                    if include_best_case {
+                        Some(HCM::new_best_case(host, &mut rng))
+                    } else {
+                        Some(HCM::new_random_case(host, &mut rng, input))
+                    }
                 }
+                2 => Some(HCM::new_worst_case(host, &mut rng, large_input)),
+                n if n < iterations => Some(HCM::new_random_case(host, &mut rng, input)),
                 _ => None,
             }
         },

soroban-env-host/benches/common/mod.rs

@@ -1,16 +1,19 @@
 #![allow(dead_code)]
 
 mod cost_types;
+mod experimental;
 mod measure;
 mod modelfit;
 mod util;
 
 use cost_types::*;
+use experimental::*;
 pub use measure::*;
 pub use modelfit::*;
 
+use soroban_env_common::xdr::Name;
 use soroban_env_host::{
-    cost_runner::{CostRunner, WasmInsnType},
+    cost_runner::{CostRunner, CostType, WasmInsnType},
     xdr::ContractCostType,
 };
 use std::collections::BTreeMap;
@@ -29,54 +32,42 @@ fn get_explicit_bench_names() -> Option<Vec<String>> {
 
 fn should_run<HCM: HostCostMeasurement>() -> bool {
     if let Some(bench_names) = get_explicit_bench_names() {
-        let name = format!("{:?}", <HCM::Runner as CostRunner>::COST_TYPE)
-            .split("::")
-            .last()
-            .unwrap()
-            .to_string();
-        bench_names.iter().any(|arg| *arg == name)
-    } else {
-        true
-    }
-}
-
-fn should_run_wasm_insn(ty: WasmInsnType) -> bool {
-    if let Some(bench_names) = get_explicit_bench_names() {
-        let name = format!("{:?}", ty).split("::").last().unwrap().to_string();
+        let name = <HCM::Runner as CostRunner>::COST_TYPE.name();
         bench_names.iter().any(|arg| *arg == name)
     } else {
         true
     }
 }
 
 fn call_bench<B: Benchmark, HCM: HostCostMeasurement>(
-    params: &mut BTreeMap<ContractCostType, (FPCostModel, FPCostModel)>,
+    params: &mut BTreeMap<CostType, (FPCostModel, FPCostModel)>,
 ) -> std::io::Result<()> {
     if should_run::<HCM>() {
         params.insert(<HCM::Runner as CostRunner>::COST_TYPE, B::bench::<HCM>()?);
     }
     Ok(())
 }
 
+pub(crate) fn for_each_experimental_cost_measurement<B: Benchmark>(
+) -> std::io::Result<(FPCostModel, FPCostModel)> {
+    B::bench::<Ed25519ScalarMulMeasure>()?;
+    B::bench::<VerifyEd25519SigMeasure>()
+}
+
 pub(crate) fn for_each_host_cost_measurement<B: Benchmark>(
-) -> std::io::Result<BTreeMap<ContractCostType, (FPCostModel, FPCostModel)>> {
-    let mut params: BTreeMap<ContractCostType, (FPCostModel, FPCostModel)> = BTreeMap::new();
+) -> std::io::Result<BTreeMap<CostType, (FPCostModel, FPCostModel)>> {
+    let mut params: BTreeMap<CostType, (FPCostModel, FPCostModel)> = BTreeMap::new();
 
-    // call_bench::<B, ComputeEcdsaSecp256k1PubKeyMeasure>(&mut params)?;
     call_bench::<B, ComputeEcdsaSecp256k1SigMeasure>(&mut params)?;
     call_bench::<B, ComputeEd25519PubKeyMeasure>(&mut params)?;
     call_bench::<B, ComputeKeccak256HashMeasure>(&mut params)?;
     call_bench::<B, ComputeSha256HashMeasure>(&mut params)?;
     call_bench::<B, RecoverEcdsaSecp256k1KeyMeasure>(&mut params)?;
     call_bench::<B, VerifyEd25519SigMeasure>(&mut params)?;
     call_bench::<B, VmInstantiationMeasure>(&mut params)?;
-    // call_bench::<B, VmMemReadMeasure>(&mut params)?;
-    // call_bench::<B, VmMemWriteMeasure>(&mut params)?;
     call_bench::<B, VisitObjectMeasure>(&mut params)?;
     call_bench::<B, ValSerMeasure>(&mut params)?;
     call_bench::<B, ValDeserMeasure>(&mut params)?;
-    // call_bench::<B, MapEntryMeasure>(&mut params)?;
-    // call_bench::<B, VecEntryMeasure>(&mut params)?;
     call_bench::<B, MemCmpMeasure>(&mut params)?;
     call_bench::<B, InvokeVmFunctionMeasure>(&mut params)?;
     call_bench::<B, InvokeHostFunctionMeasure>(&mut params)?;
@@ -91,7 +82,7 @@ pub(crate) fn for_each_host_cost_measurement<B: Benchmark>(
 
     if get_explicit_bench_names().is_none() {
         for cost in ContractCostType::variants() {
-            if !params.contains_key(&cost) {
+            if !params.contains_key(&CostType::Contract(cost)) {
                 eprintln!("warning: missing cost measurement for {:?}", cost);
             }
         }
@@ -101,20 +92,17 @@ pub(crate) fn for_each_host_cost_measurement<B: Benchmark>(
 
 macro_rules! run_wasm_insn_measurement {
     ( $($HCM: ident),* ) => {
-        pub(crate) fn for_each_wasm_insn_measurement<B: Benchmark>() -> std::io::Result<BTreeMap<WasmInsnType, (FPCostModel, FPCostModel)>> {
-            let mut params: BTreeMap<WasmInsnType, (FPCostModel, FPCostModel)> = BTreeMap::new();
+        pub(crate) fn for_each_wasm_insn_measurement<B: Benchmark>() -> std::io::Result<BTreeMap<CostType, (FPCostModel, FPCostModel)>> {
+            let mut params: BTreeMap<CostType, (FPCostModel, FPCostModel)> = BTreeMap::new();
             $(
-                let ty = <$HCM as HostCostMeasurement>::Runner::INSN_TYPE;
-                if should_run_wasm_insn(ty) {
-                    eprintln!(
-                        "\nMeasuring costs for WasmInsnType::{:?}\n", ty);
-                    params.insert(ty, B::bench::<$HCM>()?);
+                if should_run::<$HCM>() {
+                    params.insert(<$HCM as HostCostMeasurement>::Runner::COST_TYPE, B::bench::<$HCM>()?);
                 }
             )*
 
             if get_explicit_bench_names().is_none() {
                 for insn in WasmInsnType::variants() {
-                    if !params.contains_key(insn) {
+                    if !params.contains_key(&CostType::Wasm(*insn)) {
                         eprintln!("warning: missing cost measurement for {:?}", insn);
                     }
                 }

soroban-env-host/benches/variation_histograms.rs

@@ -1,14 +1,14 @@
 // Run this with
-// $ cargo bench --features wasmi,testutils --bench variation_histograms -- --nocapture
+// $ cargo bench --features testutils --bench variation_histograms -- --nocapture
 mod common;
 use common::*;
 use soroban_env_host::cost_runner::CostRunner;
 
 struct LinearModelTables;
 impl Benchmark for LinearModelTables {
     fn bench<HCM: HostCostMeasurement>() -> std::io::Result<(FPCostModel, FPCostModel)> {
-        let mut measurements = measure_cost_variation::<HCM>(100)?;
-        measurements.check_range_against_baseline(HCM::Runner::COST_TYPE)?;
+        let mut measurements = measure_cost_variation::<HCM>(100, 1000, false, false)?;
+        measurements.check_range_against_baseline(&HCM::Runner::COST_TYPE)?;
         measurements.preprocess();
         measurements.report_histogram("cpu", |m| m.cpu_insns);
         measurements.report_histogram("mem", |m| m.mem_bytes);
@@ -18,6 +18,6 @@ impl Benchmark for LinearModelTables {
 
 #[cfg(all(test, any(target_os = "linux", target_os = "macos")))]
 fn main() -> std::io::Result<()> {
-    for_each_host_cost_measurement::<LinearModelTables>()?;
+    for_each_experimental_cost_measurement::<LinearModelTables>()?;
     Ok(())
 }