perf(api): More efficient gas estimation (matter-labs#2937)

## What ❔

Makes gas estimation more efficient by using heuristics similar to ones
used by Ethereum clients:

- Defines the lower gas limit bound as the gas consumed with an
effectively infinite gas limit.
- Defines an initial binary search pivot as a slightly larger value so
that in the best / average case, most of the search space is immediately
discarded.

## Why ❔

Improves gas estimation latency in the best and average case.

## Checklist

- [x] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [x] Tests for the changes have been added / updated.
- [x] Documentation comments have been added / updated.
- [x] Code has been formatted via `zk fmt` and `zk lint`.

core/bin/external_node/src/config/mod.rs

@@ -327,6 +327,10 @@ pub(crate) struct OptionalENConfig {
     /// The max possible number of gas that `eth_estimateGas` is allowed to overestimate.
     #[serde(default = "OptionalENConfig::default_estimate_gas_acceptable_overestimation")]
     pub estimate_gas_acceptable_overestimation: u32,
+    /// Enables optimizations for the binary search of the gas limit in `eth_estimateGas`. These optimizations are currently
+    /// considered experimental.
+    #[serde(default)]
+    pub estimate_gas_optimize_search: bool,
     /// The multiplier to use when suggesting gas price. Should be higher than one,
     /// otherwise if the L1 prices soar, the suggested gas price won't be sufficient to be included in block.
     #[serde(default = "OptionalENConfig::default_gas_price_scale_factor")]
@@ -558,6 +562,11 @@ impl OptionalENConfig {
                 web3_json_rpc.estimate_gas_acceptable_overestimation,
                 default_estimate_gas_acceptable_overestimation
             ),
+            estimate_gas_optimize_search: general_config
+                .api_config
+                .as_ref()
+                .map(|a| a.web3_json_rpc.estimate_gas_optimize_search)
+                .unwrap_or_default(),
             gas_price_scale_factor: load_config_or_default!(
                 general_config.api_config,
                 web3_json_rpc.gas_price_scale_factor,
@@ -1380,6 +1389,7 @@ impl From<&ExternalNodeConfig> for InternalApiConfig {
             estimate_gas_acceptable_overestimation: config
                 .optional
                 .estimate_gas_acceptable_overestimation,
+            estimate_gas_optimize_search: config.optional.estimate_gas_optimize_search,
             bridge_addresses: BridgeAddresses {
                 l1_erc20_default_bridge: config.remote.l1_erc20_bridge_proxy_addr,
                 l2_erc20_default_bridge: config.remote.l2_erc20_bridge_addr,

core/lib/config/src/configs/api.rs

@@ -169,6 +169,10 @@ pub struct Web3JsonRpcConfig {
     pub estimate_gas_scale_factor: f64,
     /// The max possible number of gas that `eth_estimateGas` is allowed to overestimate.
     pub estimate_gas_acceptable_overestimation: u32,
+    /// Enables optimizations for the binary search of the gas limit in `eth_estimateGas`. These optimizations are currently
+    /// considered experimental.
+    #[serde(default)]
+    pub estimate_gas_optimize_search: bool,
     ///  Max possible size of an ABI encoded tx (in bytes).
     pub max_tx_size: usize,
     /// Max number of cache misses during one VM execution. If the number of cache misses exceeds this value, the API server panics.
@@ -237,6 +241,7 @@ impl Web3JsonRpcConfig {
             gas_price_scale_factor: 1.2,
             estimate_gas_scale_factor: 1.2,
             estimate_gas_acceptable_overestimation: 1000,
+            estimate_gas_optimize_search: false,
             max_tx_size: 1000000,
             vm_execution_cache_misses_limit: Default::default(),
             vm_concurrency_limit: Default::default(),

core/lib/config/src/testonly.rs

@@ -79,6 +79,7 @@ impl Distribution<configs::api::Web3JsonRpcConfig> for EncodeDist {
             gas_price_scale_factor: self.sample(rng),
             estimate_gas_scale_factor: self.sample(rng),
             estimate_gas_acceptable_overestimation: self.sample(rng),
+            estimate_gas_optimize_search: self.sample(rng),
             max_tx_size: self.sample(rng),
             vm_execution_cache_misses_limit: self.sample(rng),
             vm_concurrency_limit: self.sample(rng),

core/lib/env_config/src/api.rs

@@ -69,6 +69,7 @@ mod tests {
                 estimate_gas_scale_factor: 1.0f64,
                 gas_price_scale_factor: 1.2,
                 estimate_gas_acceptable_overestimation: 1000,
+                estimate_gas_optimize_search: false,
                 max_tx_size: 1000000,
                 vm_execution_cache_misses_limit: None,
                 vm_concurrency_limit: Some(512),

core/lib/protobuf_config/src/api.rs

@@ -84,6 +84,7 @@ impl ProtoRepr for proto::Web3JsonRpc {
                 &self.estimate_gas_acceptable_overestimation,
             )
             .context("acceptable_overestimation")?,
+            estimate_gas_optimize_search: self.estimate_gas_optimize_search.unwrap_or(false),
             max_tx_size: required(&self.max_tx_size)
                 .and_then(|x| Ok((*x).try_into()?))
                 .context("max_tx_size")?,
@@ -167,6 +168,7 @@ impl ProtoRepr for proto::Web3JsonRpc {
             estimate_gas_acceptable_overestimation: Some(
                 this.estimate_gas_acceptable_overestimation,
             ),
+            estimate_gas_optimize_search: Some(this.estimate_gas_optimize_search),
             max_tx_size: Some(this.max_tx_size.try_into().unwrap()),
             vm_execution_cache_misses_limit: this
                 .vm_execution_cache_misses_limit

core/lib/protobuf_config/src/proto/config/api.proto

@@ -40,6 +40,8 @@ message Web3JsonRpc {
   repeated MaxResponseSizeOverride max_response_body_size_overrides = 31;
   repeated string api_namespaces = 32; // Optional, if empty all namespaces are available
   optional bool extended_api_tracing = 33; // optional, default false
+  optional bool estimate_gas_optimize_search = 34; // optional, default false
+
   reserved 15; reserved "l1_to_l2_transactions_compatibility_mode";
   reserved 11; reserved "request_timeout";
   reserved 12; reserved "account_pks";

core/node/api_server/Cargo.toml

@@ -56,6 +56,7 @@ tower-http = { workspace = true, features = ["cors", "metrics"] }
 lru.workspace = true
 
 [dev-dependencies]
+zk_evm_1_5_0.workspace = true
 zksync_node_genesis.workspace = true
 zksync_node_test_utils.workspace = true
 

core/node/api_server/src/execution_sandbox/storage.rs

@@ -8,7 +8,7 @@ use std::{
 use zksync_multivm::interface::storage::ReadStorage;
 use zksync_types::{
     api::state_override::{OverrideState, StateOverride},
-    get_code_key, get_nonce_key,
+    get_code_key, get_known_code_key, get_nonce_key,
     utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance},
     AccountTreeId, StorageKey, StorageValue, H256,
 };
@@ -56,6 +56,9 @@ impl<S: ReadStorage> StorageWithOverrides<S> {
                 let code_key = get_code_key(account);
                 let code_hash = code.hash();
                 self.overridden_slots.insert(code_key, code_hash);
+                let known_code_key = get_known_code_key(&code_hash);
+                self.overridden_slots
+                    .insert(known_code_key, H256::from_low_u64_be(1));
                 self.store_factory_dep(code_hash, code.clone().into_bytes());
             }
 

core/node/api_server/src/execution_sandbox/tests.rs

@@ -11,15 +11,15 @@ use zksync_node_test_utils::{create_l2_block, prepare_recovery_snapshot};
 use zksync_state::PostgresStorageCaches;
 use zksync_types::{
     api::state_override::{OverrideAccount, StateOverride},
-    fee::Fee,
     fee_model::BatchFeeInput,
-    l2::L2Tx,
-    transaction_request::PaymasterParams,
-    Address, K256PrivateKey, L2ChainId, Nonce, ProtocolVersionId, Transaction, U256,
+    K256PrivateKey, ProtocolVersionId, Transaction, U256,
 };
 
 use super::*;
-use crate::{execution_sandbox::execute::SandboxExecutor, tx_sender::SandboxExecutorOptions};
+use crate::{
+    execution_sandbox::execute::SandboxExecutor, testonly::TestAccount,
+    tx_sender::SandboxExecutorOptions,
+};
 
 #[tokio::test]
 async fn creating_block_args() {
@@ -210,7 +210,11 @@ async fn test_instantiating_vm(connection: Connection<'static, Core>, block_args
     let fee_input = BatchFeeInput::l1_pegged(55, 555);
     let (base_fee, gas_per_pubdata) =
         derive_base_fee_and_gas_per_pubdata(fee_input, ProtocolVersionId::latest().into());
-    let tx = Transaction::from(create_transfer(base_fee, gas_per_pubdata));
+    let tx = Transaction::from(K256PrivateKey::random().create_transfer(
+        0.into(),
+        base_fee,
+        gas_per_pubdata,
+    ));
 
     let (limiter, _) = VmConcurrencyLimiter::new(1);
     let vm_permit = limiter.acquire().await.unwrap();
@@ -229,27 +233,6 @@ async fn test_instantiating_vm(connection: Connection<'static, Core>, block_args
     assert!(!tx_result.result.is_failed(), "{tx_result:#?}");
 }
 
-fn create_transfer(fee_per_gas: u64, gas_per_pubdata: u64) -> L2Tx {
-    let fee = Fee {
-        gas_limit: 200_000.into(),
-        max_fee_per_gas: fee_per_gas.into(),
-        max_priority_fee_per_gas: 0_u64.into(),
-        gas_per_pubdata_limit: gas_per_pubdata.into(),
-    };
-    L2Tx::new_signed(
-        Some(Address::random()),
-        vec![],
-        Nonce(0),
-        fee,
-        U256::zero(),
-        L2ChainId::default(),
-        &K256PrivateKey::random(),
-        vec![],
-        PaymasterParams::default(),
-    )
-    .unwrap()
-}
-
 #[test_casing(2, [false, true])]
 #[tokio::test]
 async fn validating_transaction(set_balance: bool) {
@@ -270,7 +253,7 @@ async fn validating_transaction(set_balance: bool) {
     let fee_input = BatchFeeInput::l1_pegged(55, 555);
     let (base_fee, gas_per_pubdata) =
         derive_base_fee_and_gas_per_pubdata(fee_input, ProtocolVersionId::latest().into());
-    let tx = create_transfer(base_fee, gas_per_pubdata);
+    let tx = K256PrivateKey::random().create_transfer(0.into(), base_fee, gas_per_pubdata);
 
     let (limiter, _) = VmConcurrencyLimiter::new(1);
     let vm_permit = limiter.acquire().await.unwrap();

core/node/api_server/src/execution_sandbox/vm_metrics.rs

@@ -89,8 +89,18 @@ pub(crate) struct SandboxMetrics {
     pub(super) sandbox_execution_permits: Histogram<usize>,
     #[metrics(buckets = Buckets::LATENCIES)]
     submit_tx: Family<SubmitTxStage, Histogram<Duration>>,
+
+    /// Number of iterations necessary to estimate gas for a transaction.
     #[metrics(buckets = Buckets::linear(0.0..=30.0, 3.0))]
     pub estimate_gas_binary_search_iterations: Histogram<usize>,
+    /// Relative difference between the unscaled final gas estimate and the optimized lower bound. Positive if the lower bound
+    /// is (as expected) lower than the final gas estimate.
+    #[metrics(buckets = Buckets::linear(-0.05..=0.15, 0.01))]
+    pub estimate_gas_lower_bound_relative_diff: Histogram<f64>,
+    /// Relative difference between the optimistic gas limit and the unscaled final gas estimate. Positive if the optimistic gas limit
+    /// is (as expected) greater than the final gas estimate.
+    #[metrics(buckets = Buckets::linear(-0.05..=0.15, 0.01))]
+    pub estimate_gas_optimistic_gas_limit_relative_diff: Histogram<f64>,
 }
 
 impl SandboxMetrics {

core/node/api_server/src/lib.rs

@@ -4,5 +4,7 @@
 mod utils;
 pub mod execution_sandbox;
 pub mod healthcheck;
+#[cfg(test)]
+mod testonly;
 pub mod tx_sender;
 pub mod web3;