From 65a5d7b8f6890c2c08d6104dcbff9872a69b4228 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 8 Apr 2024 18:03:56 +0200 Subject: [PATCH] =?UTF-8?q?[FRAME]=C2=A0Runtime=20Omni=20Bencher=20=20(#35?= =?UTF-8?q?12)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This MR contains two major changes and some maintenance cleanup. ## 1. Free Standing Pallet Benchmark Runner Closes https://github.com/paritytech/polkadot-sdk/issues/3045, depends on your runtime exposing the `GenesisBuilderApi` (like https://github.com/paritytech/polkadot-sdk/pull/1492). Introduces a new binary crate: `frame-omni-bencher`. It allows to directly benchmark a WASM blob - without needing a node or chain spec. This makes it much easier to generate pallet weights and should allow us to remove bloaty code from the node. It should work for all FRAME runtimes that dont use 3rd party host calls or non `BlakeTwo256` block hashing (basically all polkadot parachains should work). It is 100% backwards compatible with the old CLI args, when the `v1` compatibility command is used. This is done to allow for forwards compatible addition of new commands. ### Example (full example in the Rust docs) Installing the CLI: ```sh cargo install --locked --path substrate/utils/frame/omni-bencher frame-omni-bencher --help ``` Building the Westend runtime: ```sh cargo build -p westend-runtime --release --features runtime-benchmarks ``` Benchmarking the runtime: ```sh frame-omni-bencher v1 benchmark pallet --runtime target/release/wbuild/westend-runtime/westend_runtime.compact.compressed.wasm --all ``` ## 2. Building the Benchmark Genesis State in the Runtime Closes https://github.com/paritytech/polkadot-sdk/issues/2664 This adds `--runtime` and `--genesis-builder=none|runtime|spec` arguments to the `benchmark pallet` command to make it possible to generate the genesis storage by the runtime. This can be used with both the node and the freestanding benchmark runners. It utilizes the new `GenesisBuilder` RA and depends on having https://github.com/paritytech/polkadot-sdk/pull/3412 deployed. ## 3. Simpler args for `PalletCmd::run` You can do three things here to integrate the changes into your node: - nothing: old code keeps working as before but emits a deprecated warning - delete: remove the pallet benchmarking code from your node and use the omni-bencher instead - patch: apply the patch below and keep using as currently. This emits a deprecated warning at runtime, since it uses the old way to generate a genesis state, but is the smallest change. ```patch runner.sync_run(|config| cmd - .run::, ReclaimHostFunctions>(config) + .run_with_spec::, ReclaimHostFunctions>(Some(config.chain_spec)) ) ``` ## 4. Maintenance Change - `pallet-nis` get a `BenchmarkSetup` config item to prepare its counterparty asset. - Add percent progress print when running benchmarks. - Dont immediately exit on benchmark error but try to run as many as possible and print errors last. --------- Signed-off-by: Oliver Tale-Yazdi Co-authored-by: Liam Aharon --- .gitlab/pipeline/check.yml | 4 +- .gitlab/pipeline/test.yml | 19 +- Cargo.lock | 16 + Cargo.toml | 1 + cumulus/polkadot-parachain/src/command.rs | 2 +- polkadot/cli/src/command.rs | 6 +- polkadot/runtime/common/src/auctions.rs | 7 +- .../runtime/parachains/src/configuration.rs | 27 +- .../parachains/src/hrmp/benchmarking.rs | 2 + .../parachains/src/paras/benchmarking.rs | 12 +- polkadot/runtime/rococo/src/lib.rs | 2 + prdoc/pr_3512.prdoc | 47 ++ substrate/bin/node/cli/src/command.rs | 2 +- substrate/bin/node/runtime/src/lib.rs | 19 + substrate/frame/babe/src/lib.rs | 2 +- substrate/frame/benchmarking/src/v1.rs | 14 +- .../frame/contracts/src/benchmarking/mod.rs | 4 +- .../election-provider-multi-phase/src/lib.rs | 2 +- substrate/frame/fast-unstake/src/lib.rs | 2 +- substrate/frame/nis/src/benchmarking.rs | 5 + substrate/frame/nis/src/lib.rs | 16 + substrate/frame/nis/src/mock.rs | 2 + .../utils/frame/benchmarking-cli/Cargo.toml | 2 + .../utils/frame/benchmarking-cli/README.md | 63 +- .../benchmarking-cli/src/pallet/command.rs | 760 ++++++++++++------ .../frame/benchmarking-cli/src/pallet/mod.rs | 24 +- .../benchmarking-cli/src/pallet/types.rs | 78 ++ .../benchmarking-cli/src/pallet/writer.rs | 26 +- substrate/utils/frame/omni-bencher/Cargo.toml | 21 + .../utils/frame/omni-bencher/src/command.rs | 149 ++++ .../utils/frame/omni-bencher/src/main.rs | 29 + templates/parachain/node/src/command.rs | 2 +- templates/solochain/node/src/command.rs | 4 +- 33 files changed, 1064 insertions(+), 307 deletions(-) create mode 100644 prdoc/pr_3512.prdoc create mode 100644 substrate/utils/frame/benchmarking-cli/src/pallet/types.rs create mode 100644 substrate/utils/frame/omni-bencher/Cargo.toml create mode 100644 substrate/utils/frame/omni-bencher/src/command.rs create mode 100644 substrate/utils/frame/omni-bencher/src/main.rs diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml index 4c39539f1e50..89b2c00db9b2 100644 --- a/.gitlab/pipeline/check.yml +++ b/.gitlab/pipeline/check.yml @@ -7,8 +7,8 @@ cargo-clippy: variables: RUSTFLAGS: "-D warnings" script: - - SKIP_WASM_BUILD=1 cargo clippy --all-targets --locked --workspace - - SKIP_WASM_BUILD=1 cargo clippy --all-targets --all-features --locked --workspace + - SKIP_WASM_BUILD=1 cargo clippy --all-targets --locked --workspace --quiet + - SKIP_WASM_BUILD=1 cargo clippy --all-targets --all-features --locked --workspace --quiet check-try-runtime: stage: check diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 9db89b923009..af16f5d2de7f 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -322,7 +322,24 @@ quick-benchmarks: WASM_BUILD_NO_COLOR: 1 WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings" script: - - time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + - time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks --quiet -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet + +quick-benchmarks-omni: + stage: test + extends: + - .docker-env + - .common-refs + - .run-immediately + variables: + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-C debug-assertions" + RUST_BACKTRACE: "full" + WASM_BUILD_NO_COLOR: 1 + WASM_BUILD_RUSTFLAGS: "-C debug-assertions" + script: + - time cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks + - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet test-frame-examples-compile-to-wasm: # into one job diff --git a/Cargo.lock b/Cargo.lock index eb624d0ffa2d..db3ce694aa29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5503,6 +5503,7 @@ dependencies = [ "rand", "rand_pcg", "sc-block-builder", + "sc-chain-spec", "sc-cli", "sc-client-api", "sc-client-db", @@ -5516,6 +5517,7 @@ dependencies = [ "sp-core", "sp-database", "sp-externalities 0.25.0", + "sp-genesis-builder", "sp-inherents", "sp-io", "sp-keystore", @@ -5628,6 +5630,20 @@ dependencies = [ "serde", ] +[[package]] +name = "frame-omni-bencher" +version = "0.1.0" +dependencies = [ + "clap 4.5.3", + "cumulus-primitives-proof-size-hostfunction", + "env_logger 0.11.3", + "frame-benchmarking-cli", + "log", + "sc-cli", + "sp-runtime", + "sp-statement-store", +] + [[package]] name = "frame-remote-externalities" version = "0.35.0" diff --git a/Cargo.toml b/Cargo.toml index f81bc8d10e55..8e6fdb4b8bbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -494,6 +494,7 @@ members = [ "substrate/utils/frame/frame-utilities-cli", "substrate/utils/frame/generate-bags", "substrate/utils/frame/generate-bags/node-runtime", + "substrate/utils/frame/omni-bencher", "substrate/utils/frame/remote-externalities", "substrate/utils/frame/rpc/client", "substrate/utils/frame/rpc/state-trie-migration-rpc", diff --git a/cumulus/polkadot-parachain/src/command.rs b/cumulus/polkadot-parachain/src/command.rs index 514968f7fbb7..c0bd7acfebf8 100644 --- a/cumulus/polkadot-parachain/src/command.rs +++ b/cumulus/polkadot-parachain/src/command.rs @@ -568,7 +568,7 @@ pub fn run() -> Result<()> { match cmd { BenchmarkCmd::Pallet(cmd) => if cfg!(feature = "runtime-benchmarks") { - runner.sync_run(|config| cmd.run::, ReclaimHostFunctions>(config)) + runner.sync_run(|config| cmd.run_with_spec::, ReclaimHostFunctions>(Some(config.chain_spec))) } else { Err("Benchmarking wasn't enabled when building the node. \ You can enable it with `--features runtime-benchmarks`." diff --git a/polkadot/cli/src/command.rs b/polkadot/cli/src/command.rs index f71891ecde34..b163a2c1367d 100644 --- a/polkadot/cli/src/command.rs +++ b/polkadot/cli/src/command.rs @@ -451,8 +451,10 @@ pub fn run() -> Result<()> { if cfg!(feature = "runtime-benchmarks") { runner.sync_run(|config| { - cmd.run::, ()>(config) - .map_err(|e| Error::SubstrateCli(e)) + cmd.run_with_spec::, ()>( + Some(config.chain_spec), + ) + .map_err(|e| Error::SubstrateCli(e)) }) } else { Err(sc_cli::Error::Input( diff --git a/polkadot/runtime/common/src/auctions.rs b/polkadot/runtime/common/src/auctions.rs index 6914fef99d52..00c2da3016d7 100644 --- a/polkadot/runtime/common/src/auctions.rs +++ b/polkadot/runtime/common/src/auctions.rs @@ -1899,10 +1899,13 @@ mod benchmarking { // Trigger epoch change for new random number value: { + pallet_babe::EpochStart::::set((Zero::zero(), u32::MAX.into())); pallet_babe::Pallet::::on_initialize(duration + now + T::EndingPeriod::get()); let authorities = pallet_babe::Pallet::::authorities(); - let next_authorities = authorities.clone(); - pallet_babe::Pallet::::enact_epoch_change(authorities, next_authorities, None); + // Check for non empty authority set since it otherwise emits a No-OP warning. + if !authorities.is_empty() { + pallet_babe::Pallet::::enact_epoch_change(authorities.clone(), authorities, None); + } } }: { diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index e1246fb88975..33061b789dc4 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -231,7 +231,7 @@ pub struct HostConfiguration { impl> Default for HostConfiguration { fn default() -> Self { - Self { + let ret = Self { async_backing_params: AsyncBackingParams { max_candidate_depth: 0, allowed_ancestry_len: 0, @@ -270,7 +270,30 @@ impl> Default for HostConfiguration> HostConfiguration { + /// Mutate the values of self to be good estimates for benchmarking. + /// + /// The values do not need to be worst-case, since the benchmarking logic extrapolates. They + /// should be a bit more than usually expected. + fn with_benchmarking_default(mut self) -> Self { + self.max_head_data_size = self.max_head_data_size.max(1 << 20); + self.max_downward_message_size = self.max_downward_message_size.max(1 << 16); + self.hrmp_channel_max_capacity = self.hrmp_channel_max_capacity.max(1000); + self.hrmp_channel_max_message_size = self.hrmp_channel_max_message_size.max(1 << 16); + self.hrmp_max_parachain_inbound_channels = + self.hrmp_max_parachain_inbound_channels.max(100); + self.hrmp_max_parachain_outbound_channels = + self.hrmp_max_parachain_outbound_channels.max(100); + self } } diff --git a/polkadot/runtime/parachains/src/hrmp/benchmarking.rs b/polkadot/runtime/parachains/src/hrmp/benchmarking.rs index c6baf2f30cf5..0251811d4960 100644 --- a/polkadot/runtime/parachains/src/hrmp/benchmarking.rs +++ b/polkadot/runtime/parachains/src/hrmp/benchmarking.rs @@ -76,6 +76,8 @@ where let deposit: BalanceOf = config.hrmp_sender_deposit.unique_saturated_into(); let capacity = config.hrmp_channel_max_capacity; let message_size = config.hrmp_channel_max_message_size; + assert!(message_size > 0, "Invalid genesis for benchmarking"); + assert!(capacity > 0, "Invalid genesis for benchmarking"); let sender: ParaId = from.into(); let sender_origin: crate::Origin = from.into(); diff --git a/polkadot/runtime/parachains/src/paras/benchmarking.rs b/polkadot/runtime/parachains/src/paras/benchmarking.rs index 56e6ff153c15..77f6a9edddc8 100644 --- a/polkadot/runtime/parachains/src/paras/benchmarking.rs +++ b/polkadot/runtime/parachains/src/paras/benchmarking.rs @@ -82,7 +82,7 @@ fn generate_disordered_actions_queue() { benchmarks! { force_set_current_code { - let c in 1 .. MAX_CODE_SIZE; + let c in MIN_CODE_SIZE .. MAX_CODE_SIZE; let new_code = ValidationCode(vec![0; c as usize]); let para_id = ParaId::from(c as u32); CurrentCodeHash::::insert(¶_id, new_code.hash()); @@ -92,7 +92,7 @@ benchmarks! { assert_last_event::(Event::CurrentCodeUpdated(para_id).into()); } force_set_current_head { - let s in 1 .. MAX_HEAD_DATA_SIZE; + let s in MIN_CODE_SIZE .. MAX_HEAD_DATA_SIZE; let new_head = HeadData(vec![0; s as usize]); let para_id = ParaId::from(1000); }: _(RawOrigin::Root, para_id, new_head) @@ -104,7 +104,7 @@ benchmarks! { let context = BlockNumberFor::::from(1000u32); }: _(RawOrigin::Root, para_id, context) force_schedule_code_upgrade { - let c in 1 .. MAX_CODE_SIZE; + let c in MIN_CODE_SIZE .. MAX_CODE_SIZE; let new_code = ValidationCode(vec![0; c as usize]); let para_id = ParaId::from(c as u32); let block = BlockNumberFor::::from(c); @@ -114,7 +114,7 @@ benchmarks! { assert_last_event::(Event::CodeUpgradeScheduled(para_id).into()); } force_note_new_head { - let s in 1 .. MAX_HEAD_DATA_SIZE; + let s in MIN_CODE_SIZE .. MAX_HEAD_DATA_SIZE; let para_id = ParaId::from(1000); let new_head = HeadData(vec![0; s as usize]); let old_code_hash = ValidationCode(vec![0]).hash(); @@ -126,7 +126,7 @@ benchmarks! { generate_disordered_pruning::(); Pallet::::schedule_code_upgrade( para_id, - ValidationCode(vec![0]), + ValidationCode(vec![0u8; MIN_CODE_SIZE as usize]), expired, &config, UpgradeStrategy::SetGoAheadSignal, @@ -145,7 +145,7 @@ benchmarks! { } add_trusted_validation_code { - let c in 1 .. MAX_CODE_SIZE; + let c in MIN_CODE_SIZE .. MAX_CODE_SIZE; let new_code = ValidationCode(vec![0; c as usize]); pvf_check::prepare_bypassing_bench::(new_code.clone()); diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 0187a94796e8..d33c58007843 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1154,6 +1154,8 @@ impl pallet_nis::Config for Runtime { type MaxIntakeWeight = MaxIntakeWeight; type ThawThrottle = ThawThrottle; type RuntimeHoldReason = RuntimeHoldReason; + #[cfg(feature = "runtime-benchmarks")] + type BenchmarkSetup = (); } parameter_types! { diff --git a/prdoc/pr_3512.prdoc b/prdoc/pr_3512.prdoc new file mode 100644 index 000000000000..d44e1ca1e296 --- /dev/null +++ b/prdoc/pr_3512.prdoc @@ -0,0 +1,47 @@ +title: "[FRAME] Introduce Runtime Omni Bencher" + +doc: + - audience: Node Dev + description: | + Introduces a new freestanding binary; the `frame-omni-bencher`. This can be used as alternative to the node-integrated `benchmark pallet` command. It currently only supports pallet benchmarking. + + The optional change to integrate this MR is in the node. The `run` function is now deprecated in favour or `run_with_spec`. This should be rather easy to integrate: + + ```patch + runner.sync_run(|config| cmd + - .run::, ReclaimHostFunctions>(config) + + .run_with_spec::, ReclaimHostFunctions>(Some(config.chain_spec)) + ) + ``` + + Additionally, a new `--runtime` CLI arg was introduced to the `benchmark pallet` command. It allows to generate the genesis state directly by the runtime, instead of using a spec file. + +crates: + - name: pallet-nis + bump: major + - name: frame-benchmarking-cli + bump: major + - name: polkadot-parachain-bin + bump: patch + - name: polkadot-cli + bump: patch + - name: polkadot-runtime-common + bump: patch + - name: polkadot-runtime-parachains + bump: patch + - name: rococo-runtime + bump: patch + - name: staging-node-cli + bump: patch + - name: kitchensink-runtime + bump: patch + - name: pallet-babe + bump: patch + - name: frame-benchmarking + bump: patch + - name: pallet-election-provider-multi-phase + bump: patch + - name: pallet-fast-unstake + bump: patch + - name: pallet-contracts + bump: patch diff --git a/substrate/bin/node/cli/src/command.rs b/substrate/bin/node/cli/src/command.rs index 964517320286..d37325c7187e 100644 --- a/substrate/bin/node/cli/src/command.rs +++ b/substrate/bin/node/cli/src/command.rs @@ -107,7 +107,7 @@ pub fn run() -> Result<()> { ) } - cmd.run::, sp_statement_store::runtime_api::HostFunctions>(config) + cmd.run_with_spec::, sp_statement_store::runtime_api::HostFunctions>(Some(config.chain_spec)) }, BenchmarkCmd::Block(cmd) => { // ensure that we keep the task manager alive diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index 76799e95a8f2..e3d93b953fc7 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -1759,6 +1759,25 @@ impl pallet_nis::Config for Runtime { type MaxIntakeWeight = MaxIntakeWeight; type ThawThrottle = ThawThrottle; type RuntimeHoldReason = RuntimeHoldReason; + #[cfg(feature = "runtime-benchmarks")] + type BenchmarkSetup = SetupAsset; +} + +#[cfg(feature = "runtime-benchmarks")] +pub struct SetupAsset; +#[cfg(feature = "runtime-benchmarks")] +impl pallet_nis::BenchmarkSetup for SetupAsset { + fn create_counterpart_asset() { + let owner = AccountId::from([0u8; 32]); + // this may or may not fail depending on if the chain spec or runtime genesis is used. + let _ = Assets::force_create( + RuntimeOrigin::root(), + 9u32.into(), + sp_runtime::MultiAddress::Id(owner), + true, + 1, + ); + } } parameter_types! { diff --git a/substrate/frame/babe/src/lib.rs b/substrate/frame/babe/src/lib.rs index 5fb107dde3ba..686ba6ec2d63 100644 --- a/substrate/frame/babe/src/lib.rs +++ b/substrate/frame/babe/src/lib.rs @@ -283,7 +283,7 @@ pub mod pallet { /// entropy was fixed (i.e. it was known to chain observers). Since epochs are defined in /// slots, which may be skipped, the block numbers may not line up with the slot numbers. #[pallet::storage] - pub(super) type EpochStart = + pub type EpochStart = StorageValue<_, (BlockNumberFor, BlockNumberFor), ValueQuery>; /// How late the current block is compared to its parent. diff --git a/substrate/frame/benchmarking/src/v1.rs b/substrate/frame/benchmarking/src/v1.rs index b2449db3d67d..ecebbfa91052 100644 --- a/substrate/frame/benchmarking/src/v1.rs +++ b/substrate/frame/benchmarking/src/v1.rs @@ -850,8 +850,8 @@ macro_rules! impl_bench_name_tests { if !($extra) { let disabled = $crate::__private::vec![ $( stringify!($names_extra).as_ref() ),* ]; if disabled.contains(&stringify!($name)) { - $crate::__private::log::error!( - "INFO: extra benchmark skipped - {}", + $crate::__private::log::debug!( + "extra benchmark skipped - {}", stringify!($name), ); return (); @@ -874,21 +874,21 @@ macro_rules! impl_bench_name_tests { $crate::BenchmarkError::Override(_) => { // This is still considered a success condition. $crate::__private::log::error!( - "WARNING: benchmark error overridden - {}", + "benchmark error overridden - {}", stringify!($name), ); }, $crate::BenchmarkError::Skip => { // This is considered a success condition. - $crate::__private::log::error!( - "WARNING: benchmark error skipped - {}", + $crate::__private::log::debug!( + "benchmark skipped - {}", stringify!($name), ); }, $crate::BenchmarkError::Weightless => { // This is considered a success condition. - $crate::__private::log::error!( - "WARNING: benchmark weightless skipped - {}", + $crate::__private::log::debug!( + "benchmark weightless skipped - {}", stringify!($name), ); } diff --git a/substrate/frame/contracts/src/benchmarking/mod.rs b/substrate/frame/contracts/src/benchmarking/mod.rs index 9fb107537ba0..523d93de2d47 100644 --- a/substrate/frame/contracts/src/benchmarking/mod.rs +++ b/substrate/frame/contracts/src/benchmarking/mod.rs @@ -332,7 +332,7 @@ mod benchmarks { #[benchmark(pov_mode = Measured)] fn migration_noop() { let version = LATEST_MIGRATION_VERSION; - assert_eq!(StorageVersion::get::>(), version); + StorageVersion::new(version).put::>(); #[block] { Migration::::migrate(Weight::MAX); @@ -358,7 +358,7 @@ mod benchmarks { #[benchmark(pov_mode = Measured)] fn on_runtime_upgrade_noop() { let latest_version = LATEST_MIGRATION_VERSION; - assert_eq!(StorageVersion::get::>(), latest_version); + StorageVersion::new(latest_version).put::>(); #[block] { as frame_support::traits::OnRuntimeUpgrade>::on_runtime_upgrade(); diff --git a/substrate/frame/election-provider-multi-phase/src/lib.rs b/substrate/frame/election-provider-multi-phase/src/lib.rs index 11577cd35262..63b4c49cdfe4 100644 --- a/substrate/frame/election-provider-multi-phase/src/lib.rs +++ b/substrate/frame/election-provider-multi-phase/src/lib.rs @@ -931,7 +931,7 @@ pub mod pallet { .expect(error_message); // Store the newly received solution. - log!(info, "queued unsigned solution with score {:?}", ready.score); + log!(debug, "queued unsigned solution with score {:?}", ready.score); let ejected_a_solution = >::exists(); >::put(ready); Self::deposit_event(Event::SolutionStored { diff --git a/substrate/frame/fast-unstake/src/lib.rs b/substrate/frame/fast-unstake/src/lib.rs index 04a50543bcc9..8ba306201310 100644 --- a/substrate/frame/fast-unstake/src/lib.rs +++ b/substrate/frame/fast-unstake/src/lib.rs @@ -560,7 +560,7 @@ pub mod pallet { if !remaining.is_zero() { Self::halt("not enough balance to unreserve"); } else { - log!(info, "unstaked {:?}, outcome: {:?}", stash, result); + log!(debug, "unstaked {:?}, outcome: {:?}", stash, result); Self::deposit_event(Event::::Unstaked { stash, result }); } }; diff --git a/substrate/frame/nis/src/benchmarking.rs b/substrate/frame/nis/src/benchmarking.rs index 0cc9e7421d0e..f6a83b78d518 100644 --- a/substrate/frame/nis/src/benchmarking.rs +++ b/substrate/frame/nis/src/benchmarking.rs @@ -106,6 +106,7 @@ benchmarks! { } fund_deficit { + T::BenchmarkSetup::create_counterpart_asset(); let origin = T::FundOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?; let caller: T::AccountId = whitelisted_caller(); @@ -126,6 +127,7 @@ benchmarks! { } communify { + T::BenchmarkSetup::create_counterpart_asset(); let caller: T::AccountId = whitelisted_caller(); let bid = T::MinBid::get().max(One::one()) * 100u32.into(); let ed = T::Currency::minimum_balance(); @@ -139,6 +141,7 @@ benchmarks! { } privatize { + T::BenchmarkSetup::create_counterpart_asset(); let caller: T::AccountId = whitelisted_caller(); let bid = T::MinBid::get().max(One::one()); let ed = T::Currency::minimum_balance(); @@ -153,6 +156,7 @@ benchmarks! { } thaw_private { + T::BenchmarkSetup::create_counterpart_asset(); let whale: T::AccountId = account("whale", 0, SEED); let caller: T::AccountId = whitelisted_caller(); let bid = T::MinBid::get().max(One::one()); @@ -170,6 +174,7 @@ benchmarks! { } thaw_communal { + T::BenchmarkSetup::create_counterpart_asset(); let whale: T::AccountId = account("whale", 0, SEED); let caller: T::AccountId = whitelisted_caller(); let bid = T::MinBid::get().max(One::one()); diff --git a/substrate/frame/nis/src/lib.rs b/substrate/frame/nis/src/lib.rs index 7655cd1a8243..63287f6a1802 100644 --- a/substrate/frame/nis/src/lib.rs +++ b/substrate/frame/nis/src/lib.rs @@ -155,6 +155,18 @@ impl Convert for NoCounterpart { } } +/// Setup the empty genesis state for benchmarking. +pub trait BenchmarkSetup { + /// Create the counterpart asset. Should panic on error. + /// + /// This is called prior to assuming that a counterpart balance exists. + fn create_counterpart_asset(); +} + +impl BenchmarkSetup for () { + fn create_counterpart_asset() {} +} + #[frame_support::pallet] pub mod pallet { use super::{FunInspect, FunMutate}; @@ -297,6 +309,10 @@ pub mod pallet { /// The maximum proportion which may be thawed and the period over which it is reset. #[pallet::constant] type ThawThrottle: Get<(Perquintill, BlockNumberFor)>; + + /// Setup the state for benchmarking. + #[cfg(feature = "runtime-benchmarks")] + type BenchmarkSetup: crate::BenchmarkSetup; } #[pallet::pallet] diff --git a/substrate/frame/nis/src/mock.rs b/substrate/frame/nis/src/mock.rs index 33464db34c30..f3320a306df7 100644 --- a/substrate/frame/nis/src/mock.rs +++ b/substrate/frame/nis/src/mock.rs @@ -121,6 +121,8 @@ impl pallet_nis::Config for Test { type MinReceipt = MinReceipt; type ThawThrottle = ThawThrottle; type RuntimeHoldReason = RuntimeHoldReason; + #[cfg(feature = "runtime-benchmarks")] + type BenchmarkSetup = (); } // This function basically just builds a genesis storage key/value store according to diff --git a/substrate/utils/frame/benchmarking-cli/Cargo.toml b/substrate/utils/frame/benchmarking-cli/Cargo.toml index b7fbb24b1a09..92169484c928 100644 --- a/substrate/utils/frame/benchmarking-cli/Cargo.toml +++ b/substrate/utils/frame/benchmarking-cli/Cargo.toml @@ -37,6 +37,7 @@ frame-benchmarking = { path = "../../../frame/benchmarking" } frame-support = { path = "../../../frame/support" } frame-system = { path = "../../../frame/system" } sc-block-builder = { path = "../../../client/block-builder" } +sc-chain-spec = { path = "../../../client/chain-spec", default-features = false } sc-cli = { path = "../../../client/cli", default-features = false } sc-client-api = { path = "../../../client/api" } sc-client-db = { path = "../../../client/db", default-features = false } @@ -48,6 +49,7 @@ sp-blockchain = { path = "../../../primitives/blockchain" } sp-core = { path = "../../../primitives/core" } sp-database = { path = "../../../primitives/database" } sp-externalities = { path = "../../../primitives/externalities" } +sp-genesis-builder = { path = "../../../primitives/genesis-builder" } sp-inherents = { path = "../../../primitives/inherents" } sp-keystore = { path = "../../../primitives/keystore" } sp-runtime = { path = "../../../primitives/runtime" } diff --git a/substrate/utils/frame/benchmarking-cli/README.md b/substrate/utils/frame/benchmarking-cli/README.md index 27673ea9580d..5deb5098b5bf 100644 --- a/substrate/utils/frame/benchmarking-cli/README.md +++ b/substrate/utils/frame/benchmarking-cli/README.md @@ -1,13 +1,23 @@ -# The Benchmarking CLI +# The FRAME Benchmarking CLI -This crate contains commands to benchmark various aspects of Substrate and the hardware. -All commands are exposed by the Substrate node but can be exposed by any Substrate client. +This crate contains commands to benchmark various aspects of Substrate and the hardware. The goal is to have a comprehensive suite of benchmarks that cover all aspects of Substrate and the hardware that its -running on. +running on. +There exist fundamentally two ways to use this crate. A node-integrated CLI version, and a freestanding CLI. If you are +only interested in pallet benchmarking, then skip ahead to the [Freestanding CLI](#freestanding-cli). + +# Node Integrated CLI + +Mostly all Substrate nodes will expose some commands for benchmarking. You can refer to the `staging-node-cli` crate as +an example on how to integrate those. Note that for solely benchmarking pallets, the freestanding CLI is more suitable. + +## Usage + +Here we invoke the root command on the `staging-node-cli`. Most Substrate nodes should have a similar output, depending +on their integration of these commands. -Invoking the root benchmark command prints a help menu: ```sh -$ cargo run --profile=production -- benchmark +$ cargo run -p staging-node-cli --profile=production --features=runtime-benchmarks -- benchmark Sub-commands concerned with benchmarking. @@ -31,7 +41,46 @@ use `--release`. For the final results the `production` profile and reference hardware should be used, otherwise the results are not comparable. -The sub-commands are explained in depth here: +# Freestanding CLI + +The freestanding is a standalone CLI that does not rely on any node integration. It can be used to benchmark pallets of +any FRAME runtime that does not utilize 3rd party host functions. +It currently only supports pallet benchmarking, since the other commands still rely on a node. + +## Installation + +Installing from local source repository: + +```sh +cargo install --locked --path substrate/utils/frame/omni-bencher --profile=production +``` + +## Usage + +The exposed pallet sub-command is identical as the node-integrated CLI. The only difference is that it needs to be prefixed +with a `v1` to ensure drop-in compatibility. + +First we need to ensure that there is a runtime available. As example we will build the Westend runtime: + +```sh +cargo build -p westend-runtime --profile production --features runtime-benchmarks +``` + +Now the benchmarking can be started with: + +```sh +frame-omni-bencher v1 \ + benchmark pallet \ + --runtime target/release/wbuild/westend-runtime/westend-runtime.compact.compressed.wasm \ + --pallet "pallet_balances" --extrinsic "" +``` + +For the exact arguments of the `pallet` command, please refer to the [pallet] sub-module. + +# Commands + +The sub-commands of both CLIs have the same semantics and are documented in their respective sub-modules: + - [block] Compare the weight of a historic block to its actual resource usage - [machine] Gauges the speed of the hardware - [overhead] Creates weight files for the *Block*- and *Extrinsic*-base weights diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs index 5fbfc0530bb3..305a9b960b98 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -15,7 +15,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{writer, ListOutput, PalletCmd}; +use super::{ + types::{ComponentRange, ComponentRangeMap}, + writer, ListOutput, PalletCmd, +}; +use crate::pallet::{types::FetchedCode, GenesisBuilder}; use codec::{Decode, Encode}; use frame_benchmarking::{ Analysis, BenchmarkBatch, BenchmarkBatchSplitResults, BenchmarkList, BenchmarkParameter, @@ -23,23 +27,25 @@ use frame_benchmarking::{ }; use frame_support::traits::StorageInfo; use linked_hash_map::LinkedHashMap; -use sc_cli::{execution_method_from_cli, CliConfiguration, Result, SharedParams}; +use sc_chain_spec::json_patch::merge as json_merge; +use sc_cli::{execution_method_from_cli, ChainSpec, CliConfiguration, Result, SharedParams}; use sc_client_db::BenchmarkingState; use sc_executor::{HeapAllocStrategy, WasmExecutor, DEFAULT_HEAP_ALLOC_STRATEGY}; -use sc_service::Configuration; -use serde::Serialize; use sp_core::{ offchain::{ testing::{TestOffchainExt, TestTransactionPoolExt}, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt, }, - traits::{CallContext, ReadRuntimeVersionExt}, + traits::{CallContext, CodeExecutor, ReadRuntimeVersionExt, WrappedRuntimeCode}, }; use sp_externalities::Extensions; +use sp_genesis_builder::{PresetId, Result as GenesisBuildResult}; use sp_keystore::{testing::MemoryKeystore, KeystoreExt}; use sp_runtime::traits::Hash; -use sp_state_machine::StateMachine; +use sp_state_machine::{OverlayedChanges, StateMachine}; +use sp_wasm_interface::HostFunctions; use std::{ + borrow::Cow, collections::{BTreeMap, BTreeSet, HashMap}, fmt::Debug, fs, @@ -50,17 +56,6 @@ use std::{ /// Logging target const LOG_TARGET: &'static str = "polkadot_sdk_frame::benchmark::pallet"; -/// The inclusive range of a component. -#[derive(Serialize, Debug, Clone, Eq, PartialEq)] -pub(crate) struct ComponentRange { - /// Name of the component. - name: String, - /// Minimal valid value of the component. - min: u32, - /// Maximal valid value of the component. - max: u32, -} - /// How the PoV size of a storage item should be estimated. #[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy)] pub enum PovEstimationMode { @@ -87,7 +82,15 @@ impl FromStr for PovEstimationMode { /// Maps (pallet, benchmark) -> ((pallet, storage) -> PovEstimationMode) pub(crate) type PovModesMap = - HashMap<(Vec, Vec), HashMap<(String, String), PovEstimationMode>>; + HashMap<(String, String), HashMap<(String, String), PovEstimationMode>>; + +#[derive(Debug, Clone)] +struct SelectedBenchmark { + pallet: String, + extrinsic: String, + components: Vec<(BenchmarkParameter, u32, u32)>, + pov_modes: Vec<(String, String)>, +} // This takes multiple benchmark batches and combines all the results where the pallet, instance, // and benchmark are the same. @@ -145,41 +148,56 @@ This could mean that you either did not build the node correctly with the \ `--features runtime-benchmarks` flag, or the chain spec that you are using was \ not created by a node that was compiled with the flag"; +/// When the runtime could not build the genesis storage. +const ERROR_CANNOT_BUILD_GENESIS: &str = "The runtime returned \ +an error when trying to build the genesis storage. Please ensure that all pallets \ +define a genesis config that can be built. This can be tested with: \ +https://github.com/paritytech/polkadot-sdk/pull/3412"; + +/// Warn when using the chain spec to generate the genesis state. +const WARN_SPEC_GENESIS_CTOR: &'static str = "Using the chain spec instead of the runtime to \ +generate the genesis state is deprecated. Please remove the `--chain`/`--dev`/`--local` argument, \ +point `--runtime` to your runtime blob and set `--genesis-builder=runtime`. This warning may \ +become a hard error any time after December 2024."; + +/// The preset that we expect to find in the GenesisBuilder runtime API. +const GENESIS_PRESET: &str = "development"; + impl PalletCmd { /// Runs the command and benchmarks a pallet. - pub fn run(&self, config: Configuration) -> Result<()> + #[deprecated( + note = "`run` will be removed after December 2024. Use `run_with_spec` instead or \ + completely remove the code and use the `frame-benchmarking-cli` instead (see \ + https://github.com/paritytech/polkadot-sdk/pull/3512)." + )] + pub fn run(&self, config: sc_service::Configuration) -> Result<()> where Hasher: Hash, - ExtraHostFunctions: sp_wasm_interface::HostFunctions, + ExtraHostFunctions: HostFunctions, { + self.run_with_spec::(Some(config.chain_spec)) + } + + /// Runs the pallet benchmarking command. + pub fn run_with_spec( + &self, + chain_spec: Option>, + ) -> Result<()> + where + Hasher: Hash, + ExtraHostFunctions: HostFunctions, + { + self.check_args()?; let _d = self.execution.as_ref().map(|exec| { - // We print the warning at the end, since there is often A LOT of output. + // We print the error at the end, since there is often A LOT of output. sp_core::defer::DeferGuard::new(move || { - log::warn!( + log::error!( target: LOG_TARGET, "⚠️ Argument `--execution` is deprecated. Its value of `{exec}` has on effect.", ) }) }); - if let Some(output_path) = &self.output { - if !output_path.is_dir() && output_path.file_name().is_none() { - return Err("Output file or path is invalid!".into()) - } - } - - if let Some(header_file) = &self.header { - if !header_file.is_file() { - return Err("Header file is invalid!".into()) - }; - } - - if let Some(handlebars_template_file) = &self.template { - if !handlebars_template_file.is_file() { - return Err("Handlebars template file is invalid!".into()) - }; - } - if let Some(json_input) = &self.json_input { let raw_data = match std::fs::read(json_input) { Ok(raw_data) => raw_data, @@ -194,16 +212,10 @@ impl PalletCmd { return self.output_from_results(&batches) } - let spec = config.chain_spec; - let pallet = self.pallet.clone().unwrap_or_default(); - let pallet = pallet.as_bytes(); - - let extrinsic = self.extrinsic.clone().unwrap_or_default(); - let extrinsic_split: Vec<&str> = extrinsic.split(',').collect(); - let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect(); + let (genesis_storage, genesis_changes) = + self.genesis_storage::(&chain_spec)?; + let mut changes = genesis_changes.clone(); - let genesis_storage = spec.build_storage()?; - let mut changes = Default::default(); let cache_size = Some(self.database_cache_size as usize); let state_with_tracking = BenchmarkingState::::new( genesis_storage.clone(), @@ -225,11 +237,10 @@ impl PalletCmd { let method = execution_method_from_cli(self.wasm_method, self.wasmtime_instantiation_strategy); - let heap_pages = - self.heap_pages - .map_or(DEFAULT_HEAP_ALLOC_STRATEGY, |p| HeapAllocStrategy::Static { - extra_pages: p as _, - }); + let state = &state_without_tracking; + let runtime = self.runtime_blob(&state_without_tracking)?; + let runtime_code = runtime.code()?; + let alloc_strategy = Self::alloc_strategy(runtime_code.heap_pages); let executor = WasmExecutor::<( sp_io::SubstrateHostFunctions, @@ -237,91 +248,30 @@ impl PalletCmd { ExtraHostFunctions, )>::builder() .with_execution_method(method) - .with_onchain_heap_alloc_strategy(heap_pages) - .with_offchain_heap_alloc_strategy(heap_pages) + .with_allow_missing_host_functions(self.allow_missing_host_functions) + .with_onchain_heap_alloc_strategy(alloc_strategy) + .with_offchain_heap_alloc_strategy(alloc_strategy) .with_max_runtime_instances(2) .with_runtime_cache_size(2) .build(); - let extensions = || -> Extensions { - let mut extensions = Extensions::default(); - let (offchain, _) = TestOffchainExt::new(); - let (pool, _) = TestTransactionPoolExt::new(); - let keystore = MemoryKeystore::new(); - extensions.register(KeystoreExt::new(keystore)); - extensions.register(OffchainWorkerExt::new(offchain.clone())); - extensions.register(OffchainDbExt::new(offchain)); - extensions.register(TransactionPoolExt::new(pool)); - extensions.register(ReadRuntimeVersionExt::new(executor.clone())); - extensions - }; - - // Get Benchmark List - let state = &state_without_tracking; - let result = StateMachine::new( - state, - &mut changes, - &executor, - "Benchmark_benchmark_metadata", - &(self.extra).encode(), - &mut extensions(), - &sp_state_machine::backend::BackendRuntimeCode::new(state).runtime_code()?, - CallContext::Offchain, - ) - .execute() - .map_err(|e| format!("{}: {}", ERROR_METADATA_NOT_FOUND, e))?; - - let (list, storage_info) = - <(Vec, Vec) as Decode>::decode(&mut &result[..]) - .map_err(|e| format!("Failed to decode benchmark metadata: {:?}", e))?; + let (list, storage_info): (Vec, Vec) = + Self::exec_state_machine( + StateMachine::new( + state, + &mut changes, + &executor, + "Benchmark_benchmark_metadata", + &(self.extra).encode(), + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + ERROR_METADATA_NOT_FOUND, + )?; // Use the benchmark list and the user input to determine the set of benchmarks to run. - let mut benchmarks_to_run = Vec::new(); - list.iter() - .filter(|item| pallet.is_empty() || pallet == &b"*"[..] || pallet == &item.pallet[..]) - .for_each(|item| { - for benchmark in &item.benchmarks { - let benchmark_name = &benchmark.name; - if extrinsic.is_empty() || - extrinsic.as_bytes() == &b"*"[..] || - extrinsics.contains(&&benchmark_name[..]) - { - benchmarks_to_run.push(( - item.pallet.clone(), - benchmark.name.clone(), - benchmark.components.clone(), - benchmark.pov_modes.clone(), - )) - } - } - }); - // Convert `Vec` to `String` for better readability. - let benchmarks_to_run: Vec<_> = benchmarks_to_run - .into_iter() - .map(|(pallet, extrinsic, components, pov_modes)| { - let pallet_name = - String::from_utf8(pallet.clone()).expect("Encoded from String; qed"); - let extrinsic_name = - String::from_utf8(extrinsic.clone()).expect("Encoded from String; qed"); - ( - pallet, - extrinsic, - components, - pov_modes - .into_iter() - .map(|(p, s)| { - (String::from_utf8(p).unwrap(), String::from_utf8(s).unwrap()) - }) - .collect(), - pallet_name, - extrinsic_name, - ) - }) - .collect(); - - if benchmarks_to_run.is_empty() { - return Err("No benchmarks found which match your input.".into()) - } + let benchmarks_to_run = self.select_benchmarks_to_run(list)?; if let Some(list_output) = self.list { list_benchmark(benchmarks_to_run, list_output, self.no_csv_header); @@ -333,15 +283,17 @@ impl PalletCmd { let mut batches_db = Vec::new(); let mut timer = time::SystemTime::now(); // Maps (pallet, extrinsic) to its component ranges. - let mut component_ranges = HashMap::<(Vec, Vec), Vec>::new(); + let mut component_ranges = HashMap::<(String, String), Vec>::new(); let pov_modes = Self::parse_pov_modes(&benchmarks_to_run)?; + let mut failed = Vec::<(String, String)>::new(); - for (pallet, extrinsic, components, _, pallet_name, extrinsic_name) in - benchmarks_to_run.clone() + 'outer: for (i, SelectedBenchmark { pallet, extrinsic, components, .. }) in + benchmarks_to_run.clone().into_iter().enumerate() { log::info!( target: LOG_TARGET, - "Starting benchmark: {pallet_name}::{extrinsic_name}" + "[{: >3} % ] Starting benchmark: {pallet}::{extrinsic}", + (i * 100) / benchmarks_to_run.len(), ); let all_components = if components.is_empty() { vec![Default::default()] @@ -392,100 +344,127 @@ impl PalletCmd { for (s, selected_components) in all_components.iter().enumerate() { // First we run a verification if !self.no_verify { + let mut changes = genesis_changes.clone(); let state = &state_without_tracking; - let result = StateMachine::new( - state, - &mut changes, - &executor, - "Benchmark_dispatch_benchmark", - &( - &pallet, - &extrinsic, - &selected_components.clone(), - true, // run verification code - 1, // no need to do internal repeats - ) - .encode(), - &mut extensions(), - &sp_state_machine::backend::BackendRuntimeCode::new(state) - .runtime_code()?, - CallContext::Offchain, - ) - .execute() - .map_err(|e| { - format!("Error executing and verifying runtime benchmark: {}", e) - })?; // Don't use these results since verification code will add overhead. - let _batch = - , String> as Decode>::decode( - &mut &result[..], - ) - .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))? - .map_err(|e| { - format!("Benchmark {pallet_name}::{extrinsic_name} failed: {e}",) - })?; + let _batch: Vec = match Self::exec_state_machine::< + std::result::Result, String>, + _, + _, + >( + StateMachine::new( + state, + &mut changes, + &executor, + "Benchmark_dispatch_benchmark", + &( + pallet.as_bytes(), + extrinsic.as_bytes(), + &selected_components.clone(), + true, // run verification code + 1, // no need to do internal repeats + ) + .encode(), + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + "dispatch a benchmark", + ) { + Err(e) => { + log::error!("Error executing and verifying runtime benchmark: {}", e); + failed.push((pallet.clone(), extrinsic.clone())); + continue 'outer + }, + Ok(Err(e)) => { + log::error!("Error executing and verifying runtime benchmark: {}", e); + failed.push((pallet.clone(), extrinsic.clone())); + continue 'outer + }, + Ok(Ok(b)) => b, + }; } // Do one loop of DB tracking. { + let mut changes = genesis_changes.clone(); let state = &state_with_tracking; - let result = StateMachine::new( - state, // todo remove tracking - &mut changes, - &executor, - "Benchmark_dispatch_benchmark", - &( - &pallet.clone(), - &extrinsic.clone(), - &selected_components.clone(), - false, // don't run verification code for final values - self.repeat, - ) - .encode(), - &mut extensions(), - &sp_state_machine::backend::BackendRuntimeCode::new(state) - .runtime_code()?, - CallContext::Offchain, - ) - .execute() - .map_err(|e| format!("Error executing runtime benchmark: {}", e))?; - - let batch = - , String> as Decode>::decode( - &mut &result[..], - ) - .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))??; + let batch: Vec = match Self::exec_state_machine::< + std::result::Result, String>, + _, + _, + >( + StateMachine::new( + state, // todo remove tracking + &mut changes, + &executor, + "Benchmark_dispatch_benchmark", + &( + pallet.as_bytes(), + extrinsic.as_bytes(), + &selected_components.clone(), + false, // don't run verification code for final values + self.repeat, + ) + .encode(), + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + "dispatch a benchmark", + ) { + Err(e) => { + log::error!("Error executing runtime benchmark: {}", e); + failed.push((pallet.clone(), extrinsic.clone())); + continue 'outer + }, + Ok(Err(e)) => { + log::error!("Benchmark {pallet}::{extrinsic} failed: {e}",); + failed.push((pallet.clone(), extrinsic.clone())); + continue 'outer + }, + Ok(Ok(b)) => b, + }; batches_db.extend(batch); } // Finally run a bunch of loops to get extrinsic timing information. for r in 0..self.external_repeat { + let mut changes = genesis_changes.clone(); let state = &state_without_tracking; - let result = StateMachine::new( - state, // todo remove tracking - &mut changes, - &executor, - "Benchmark_dispatch_benchmark", - &( - &pallet.clone(), - &extrinsic.clone(), - &selected_components.clone(), - false, // don't run verification code for final values - self.repeat, - ) - .encode(), - &mut extensions(), - &sp_state_machine::backend::BackendRuntimeCode::new(state) - .runtime_code()?, - CallContext::Offchain, - ) - .execute() - .map_err(|e| format!("Error executing runtime benchmark: {}", e))?; - - let batch = - , String> as Decode>::decode( - &mut &result[..], - ) - .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))??; + let batch = match Self::exec_state_machine::< + std::result::Result, String>, + _, + _, + >( + StateMachine::new( + state, // todo remove tracking + &mut changes, + &executor, + "Benchmark_dispatch_benchmark", + &( + pallet.as_bytes(), + extrinsic.as_bytes(), + &selected_components.clone(), + false, // don't run verification code for final values + self.repeat, + ) + .encode(), + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + "dispatch a benchmark", + ) { + Err(e) => { + return Err(format!("Error executing runtime benchmark: {e}",).into()); + }, + Ok(Err(e)) => { + return Err( + format!("Benchmark {pallet}::{extrinsic} failed: {e}",).into() + ); + }, + Ok(Ok(b)) => b, + }; batches.extend(batch); @@ -496,7 +475,8 @@ impl PalletCmd { log::info!( target: LOG_TARGET, - "Running benchmark: {pallet_name}::{extrinsic_name}({} args) {}/{} {}/{}", + "[{: >3} % ] Running benchmark: {pallet}::{extrinsic}({} args) {}/{} {}/{}", + (i * 100) / benchmarks_to_run.len(), components.len(), s + 1, // s starts at 0. all_components.len(), @@ -509,21 +489,276 @@ impl PalletCmd { } } + assert!(batches_db.len() == batches.len() / self.external_repeat as usize); + + if !failed.is_empty() { + failed.sort(); + eprintln!( + "The following {} benchmarks failed:\n{}", + failed.len(), + failed.iter().map(|(p, e)| format!("- {p}::{e}")).collect::>().join("\n") + ); + return Err(format!("{} benchmarks failed", failed.len()).into()) + } + // Combine all of the benchmark results, so that benchmarks of the same pallet/function // are together. let batches = combine_batches(batches, batches_db); self.output(&batches, &storage_info, &component_ranges, pov_modes) } + fn select_benchmarks_to_run(&self, list: Vec) -> Result> { + let pallet = self.pallet.clone().unwrap_or_default(); + let pallet = pallet.as_bytes(); + + let extrinsic = self.extrinsic.clone().unwrap_or_default(); + let extrinsic_split: Vec<&str> = extrinsic.split(',').collect(); + let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect(); + + // Use the benchmark list and the user input to determine the set of benchmarks to run. + let mut benchmarks_to_run = Vec::new(); + list.iter() + .filter(|item| pallet.is_empty() || pallet == &b"*"[..] || pallet == &item.pallet[..]) + .for_each(|item| { + for benchmark in &item.benchmarks { + let benchmark_name = &benchmark.name; + if extrinsic.is_empty() || + extrinsic.as_bytes() == &b"*"[..] || + extrinsics.contains(&&benchmark_name[..]) + { + benchmarks_to_run.push(( + item.pallet.clone(), + benchmark.name.clone(), + benchmark.components.clone(), + benchmark.pov_modes.clone(), + )) + } + } + }); + // Convert `Vec` to `String` for better readability. + let benchmarks_to_run: Vec<_> = benchmarks_to_run + .into_iter() + .map(|(pallet, extrinsic, components, pov_modes)| { + let pallet = String::from_utf8(pallet.clone()).expect("Encoded from String; qed"); + let extrinsic = + String::from_utf8(extrinsic.clone()).expect("Encoded from String; qed"); + + SelectedBenchmark { + pallet, + extrinsic, + components, + pov_modes: pov_modes + .into_iter() + .map(|(p, s)| { + (String::from_utf8(p).unwrap(), String::from_utf8(s).unwrap()) + }) + .collect(), + } + }) + .collect(); + + if benchmarks_to_run.is_empty() { + return Err("No benchmarks found which match your input.".into()) + } + + Ok(benchmarks_to_run) + } + + /// Produce a genesis storage and genesis changes. + /// + /// It would be easier to only return one type, but there is no easy way to convert them. + // TODO: Re-write `BenchmarkingState` to not be such a clusterfuck and only accept + // `OverlayedChanges` instead of a mix between `OverlayedChanges` and `State`. But this can only + // be done once we deprecated and removed the legacy interface :( + fn genesis_storage( + &self, + chain_spec: &Option>, + ) -> Result<(sp_storage::Storage, OverlayedChanges)> { + Ok(match (self.genesis_builder, self.runtime.is_some()) { + (Some(GenesisBuilder::None), _) => Default::default(), + (Some(GenesisBuilder::Spec), _) | (None, false) => { + log::warn!("{WARN_SPEC_GENESIS_CTOR}"); + let Some(chain_spec) = chain_spec else { + return Err("No chain spec specified to generate the genesis state".into()); + }; + + let storage = chain_spec + .build_storage() + .map_err(|e| format!("{ERROR_CANNOT_BUILD_GENESIS}\nError: {e}"))?; + + (storage, Default::default()) + }, + (Some(GenesisBuilder::Runtime), _) | (None, true) => + (Default::default(), self.genesis_from_runtime::()?), + }) + } + + /// Generate the genesis changeset by the runtime API. + fn genesis_from_runtime(&self) -> Result> { + let state = BenchmarkingState::::new( + Default::default(), + Some(self.database_cache_size as usize), + false, + false, + )?; + + // Create a dummy WasmExecutor just to build the genesis storage. + let method = + execution_method_from_cli(self.wasm_method, self.wasmtime_instantiation_strategy); + let executor = WasmExecutor::<( + sp_io::SubstrateHostFunctions, + frame_benchmarking::benchmarking::HostFunctions, + F, + )>::builder() + .with_execution_method(method) + .with_allow_missing_host_functions(self.allow_missing_host_functions) + .build(); + + let runtime = self.runtime_blob(&state)?; + let runtime_code = runtime.code()?; + + // We cannot use the `GenesisConfigBuilderRuntimeCaller` here since it returns the changes + // as `Storage` item, but we need it as `OverlayedChanges`. + let genesis_json: Option> = Self::exec_state_machine( + StateMachine::new( + &state, + &mut Default::default(), + &executor, + "GenesisBuilder_get_preset", + &None::.encode(), // Use the default preset + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + "build the genesis spec", + )?; + + let Some(base_genesis_json) = genesis_json else { + return Err("GenesisBuilder::get_preset returned no data".into()) + }; + + let base_genesis_json = serde_json::from_slice::(&base_genesis_json) + .map_err(|e| format!("GenesisBuilder::get_preset returned invalid JSON: {:?}", e))?; + + let dev_genesis_json: Option> = Self::exec_state_machine( + StateMachine::new( + &state, + &mut Default::default(), + &executor, + "GenesisBuilder_get_preset", + &Some::(GENESIS_PRESET.into()).encode(), // Use the default preset + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + "build the genesis spec", + )?; + + let mut genesis_json = serde_json::Value::default(); + json_merge(&mut genesis_json, base_genesis_json); + + if let Some(dev) = dev_genesis_json { + let dev: serde_json::Value = serde_json::from_slice(&dev).map_err(|e| { + format!("GenesisBuilder::get_preset returned invalid JSON: {:?}", e) + })?; + json_merge(&mut genesis_json, dev); + } else { + log::warn!( + "Could not find genesis preset '{GENESIS_PRESET}'. Falling back to default." + ); + } + + let json_pretty_str = serde_json::to_string_pretty(&genesis_json) + .map_err(|e| format!("json to string failed: {e}"))?; + + let mut changes = Default::default(); + let build_res: GenesisBuildResult = Self::exec_state_machine( + StateMachine::new( + &state, + &mut changes, + &executor, + "GenesisBuilder_build_state", + &json_pretty_str.encode(), + &mut Extensions::default(), + &runtime_code, + CallContext::Offchain, + ), + "populate the genesis state", + )?; + + if let Err(e) = build_res { + return Err(format!("GenesisBuilder::build_state failed: {}", e).into()) + } + + Ok(changes) + } + + /// Execute a state machine and decode its return value as `R`. + fn exec_state_machine( + mut machine: StateMachine, H, Exec>, + hint: &str, + ) -> Result { + let res = machine + .execute() + .map_err(|e| format!("Could not call runtime API to {hint}: {}", e))?; + let res = R::decode(&mut &res[..]) + .map_err(|e| format!("Failed to decode runtime API result to {hint}: {:?}", e))?; + Ok(res) + } + + /// Build the extension that are available for pallet benchmarks. + fn build_extensions(exe: E) -> Extensions { + let mut extensions = Extensions::default(); + let (offchain, _) = TestOffchainExt::new(); + let (pool, _) = TestTransactionPoolExt::new(); + let keystore = MemoryKeystore::new(); + extensions.register(KeystoreExt::new(keystore)); + extensions.register(OffchainWorkerExt::new(offchain.clone())); + extensions.register(OffchainDbExt::new(offchain)); + extensions.register(TransactionPoolExt::new(pool)); + extensions.register(ReadRuntimeVersionExt::new(exe)); + extensions + } + + /// Load the runtime blob for this benchmark. + /// + /// The blob will either be loaded from the `:code` key out of the chain spec, or from a file + /// when specified with `--runtime`. + fn runtime_blob<'a, H: Hash>( + &self, + state: &'a BenchmarkingState, + ) -> Result, H>> { + if let Some(runtime) = &self.runtime { + log::info!("Loading WASM from {}", runtime.display()); + let code = fs::read(runtime)?; + let hash = sp_core::blake2_256(&code).to_vec(); + let wrapped_code = WrappedRuntimeCode(Cow::Owned(code)); + + Ok(FetchedCode::FromFile { wrapped_code, heap_pages: self.heap_pages, hash }) + } else { + log::info!("Loading WASM from genesis state"); + let state = sp_state_machine::backend::BackendRuntimeCode::new(state); + + Ok(FetchedCode::FromGenesis { state }) + } + } + + /// Allocation strategy for pallet benchmarking. + fn alloc_strategy(heap_pages: Option) -> HeapAllocStrategy { + heap_pages.map_or(DEFAULT_HEAP_ALLOC_STRATEGY, |p| HeapAllocStrategy::Static { + extra_pages: p as _, + }) + } + fn output( &self, batches: &[BenchmarkBatchSplitResults], storage_info: &[StorageInfo], - component_ranges: &HashMap<(Vec, Vec), Vec>, + component_ranges: &ComponentRangeMap, pov_modes: PovModesMap, ) -> Result<()> { // Jsonify the result and write it to a file or stdout if desired. - if !self.jsonify(&batches)? { + if !self.jsonify(&batches)? && !self.quiet { // Print the summary only if `jsonify` did not write to stdout. self.print_summary(&batches, &storage_info, pov_modes.clone()) } @@ -546,11 +781,13 @@ impl PalletCmd { /// Re-analyze a batch historic benchmark timing data. Will not take the PoV into account. fn output_from_results(&self, batches: &[BenchmarkBatchSplitResults]) -> Result<()> { - let mut component_ranges = - HashMap::<(Vec, Vec), HashMap>::new(); + let mut component_ranges = HashMap::<(String, String), HashMap>::new(); for batch in batches { let range = component_ranges - .entry((batch.pallet.clone(), batch.benchmark.clone())) + .entry(( + String::from_utf8(batch.pallet.clone()).unwrap(), + String::from_utf8(batch.benchmark.clone()).unwrap(), + )) .or_default(); for result in &batch.time_results { for (param, value) in &result.components { @@ -608,10 +845,13 @@ impl PalletCmd { ) { for batch in batches.iter() { // Print benchmark metadata + let pallet = String::from_utf8(batch.pallet.clone()).expect("Encoded from String; qed"); + let benchmark = + String::from_utf8(batch.benchmark.clone()).expect("Encoded from String; qed"); println!( "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", - String::from_utf8(batch.pallet.clone()).expect("Encoded from String; qed"), - String::from_utf8(batch.benchmark.clone()).expect("Encoded from String; qed"), + pallet, + benchmark, self.lowest_range_values, self.highest_range_values, self.steps, @@ -625,10 +865,7 @@ impl PalletCmd { if !self.no_storage_info { let mut storage_per_prefix = HashMap::, Vec>::new(); - let pov_mode = pov_modes - .get(&(batch.pallet.clone(), batch.benchmark.clone())) - .cloned() - .unwrap_or_default(); + let pov_mode = pov_modes.get(&(pallet, benchmark)).cloned().unwrap_or_default(); let comments = writer::process_storage_results( &mut storage_per_prefix, @@ -699,20 +936,11 @@ impl PalletCmd { } /// Parses the PoV modes per benchmark that were specified by the `#[pov_mode]` attribute. - fn parse_pov_modes( - benchmarks: &Vec<( - Vec, - Vec, - Vec<(BenchmarkParameter, u32, u32)>, - Vec<(String, String)>, - String, - String, - )>, - ) -> Result { + fn parse_pov_modes(benchmarks: &Vec) -> Result { use std::collections::hash_map::Entry; let mut parsed = PovModesMap::new(); - for (pallet, call, _components, pov_modes, _, _) in benchmarks { + for SelectedBenchmark { pallet, extrinsic, pov_modes, .. } in benchmarks { for (pallet_storage, mode) in pov_modes { let mode = PovEstimationMode::from_str(&mode)?; let splits = pallet_storage.split("::").collect::>(); @@ -726,7 +954,7 @@ impl PalletCmd { let (pov_pallet, pov_storage) = (splits[0], splits.get(1).unwrap_or(&"ALL")); match parsed - .entry((pallet.clone(), call.clone())) + .entry((pallet.clone(), extrinsic.clone())) .or_default() .entry((pov_pallet.to_string(), pov_storage.to_string())) { @@ -744,6 +972,33 @@ impl PalletCmd { } Ok(parsed) } + + /// Sanity check the CLI arguments. + fn check_args(&self) -> Result<()> { + if self.runtime.is_some() && self.shared_params.chain.is_some() { + unreachable!("Clap should not allow both `--runtime` and `--chain` to be provided.") + } + + if let Some(output_path) = &self.output { + if !output_path.is_dir() && output_path.file_name().is_none() { + return Err("Output file or path is invalid!".into()) + } + } + + if let Some(header_file) = &self.header { + if !header_file.is_file() { + return Err("Header file is invalid!".into()) + }; + } + + if let Some(handlebars_template_file) = &self.template { + if !handlebars_template_file.is_file() { + return Err("Handlebars template file is invalid!".into()) + }; + } + + Ok(()) + } } impl CliConfiguration for PalletCmd { @@ -761,35 +1016,28 @@ impl CliConfiguration for PalletCmd { /// List the benchmarks available in the runtime, in a CSV friendly format. fn list_benchmark( - benchmarks_to_run: Vec<( - Vec, - Vec, - Vec<(BenchmarkParameter, u32, u32)>, - Vec<(String, String)>, - String, - String, - )>, + benchmarks_to_run: Vec, list_output: ListOutput, no_csv_header: bool, ) { let mut benchmarks = BTreeMap::new(); // Sort and de-dub by pallet and function name. - benchmarks_to_run.iter().for_each(|(_, _, _, _, pallet_name, extrinsic_name)| { + benchmarks_to_run.iter().for_each(|bench| { benchmarks - .entry(pallet_name) + .entry(&bench.pallet) .or_insert_with(BTreeSet::new) - .insert(extrinsic_name); + .insert(&bench.extrinsic); }); match list_output { ListOutput::All => { if !no_csv_header { - println!("pallet,extrinsic"); + println!("pallet, extrinsic"); } for (pallet, extrinsics) in benchmarks { for extrinsic in extrinsics { - println!("{pallet},{extrinsic}"); + println!("{pallet}, {extrinsic}"); } } }, diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs index 6dc56c0724ea..d05b52f1ac87 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs @@ -16,9 +16,10 @@ // limitations under the License. mod command; +mod types; mod writer; -use crate::shared::HostInfoParams; +use crate::{pallet::types::GenesisBuilder, shared::HostInfoParams}; use clap::ValueEnum; use sc_cli::{ WasmExecutionMethod, WasmtimeInstantiationStrategy, DEFAULT_WASMTIME_INSTANTIATION_STRATEGY, @@ -166,6 +167,20 @@ pub struct PalletCmd { )] pub wasmtime_instantiation_strategy: WasmtimeInstantiationStrategy, + /// Optional runtime blob to use instead of the one from the genesis config. + #[arg(long, conflicts_with = "chain")] + pub runtime: Option, + + /// Do not fail if there are unknown but also unused host functions in the runtime. + #[arg(long)] + pub allow_missing_host_functions: bool, + + /// How to construct the genesis state. + /// + /// Uses `GenesisBuilder::Spec` by default and `GenesisBuilder::Runtime` if `runtime` is set. + #[arg(long, value_enum)] + pub genesis_builder: Option, + /// DEPRECATED: This argument has no effect. #[arg(long = "execution")] pub execution: Option, @@ -221,4 +236,11 @@ pub struct PalletCmd { /// This exists only to restore legacy behaviour. It should never actually be needed. #[arg(long)] pub unsafe_overwrite_results: bool, + + /// Do not print a summary at the end of the run. + /// + /// These summaries can be very long when benchmarking multiple pallets at once. For CI + /// use-cases, this option reduces the noise. + #[arg(long)] + quiet: bool, } diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs new file mode 100644 index 000000000000..2bb00d66560f --- /dev/null +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs @@ -0,0 +1,78 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Various types used by this crate. + +use sc_cli::Result; +use sp_core::traits::{RuntimeCode, WrappedRuntimeCode}; +use sp_runtime::traits::Hash; + +/// How the genesis state for benchmarking should be build. +#[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy)] +#[clap(rename_all = "kebab-case")] +pub enum GenesisBuilder { + /// Do not provide any genesis state. + /// + /// Benchmarks are advised to function with this, since they should setup their own required + /// state. However, to keep backwards compatibility, this is not the default. + None, + /// Let the runtime build the genesis state through its `BuildGenesisConfig` runtime API. + Runtime, + /// Use the spec file to build the genesis state. This fails when there is no spec. + Spec, +} + +/// A runtime blob that was either fetched from genesis storage or loaded from a file. +// NOTE: This enum is only needed for the annoying lifetime bounds on `RuntimeCode`. Otherwise we +// could just directly return the blob. +pub enum FetchedCode<'a, B, H> { + FromGenesis { state: sp_state_machine::backend::BackendRuntimeCode<'a, B, H> }, + FromFile { wrapped_code: WrappedRuntimeCode<'a>, heap_pages: Option, hash: Vec }, +} + +impl<'a, B, H> FetchedCode<'a, B, H> +where + H: Hash, + B: sc_client_api::StateBackend, +{ + /// The runtime blob. + pub fn code(&'a self) -> Result> { + match self { + Self::FromGenesis { state } => state.runtime_code().map_err(Into::into), + Self::FromFile { wrapped_code, heap_pages, hash } => Ok(RuntimeCode { + code_fetcher: wrapped_code, + heap_pages: *heap_pages, + hash: hash.clone(), + }), + } + } +} + +/// Maps a (pallet, benchmark) to its component ranges. +pub(crate) type ComponentRangeMap = + std::collections::HashMap<(String, String), Vec>; + +/// The inclusive range of a component. +#[derive(serde::Serialize, Debug, Clone, Eq, PartialEq)] +pub(crate) struct ComponentRange { + /// Name of the component. + pub(crate) name: String, + /// Minimal valid value of the component. + pub(crate) min: u32, + /// Maximal valid value of the component. + pub(crate) max: u32, +} diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs index bd4b65d8a2e3..df7d81b2822e 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs @@ -28,7 +28,10 @@ use itertools::Itertools; use serde::Serialize; use crate::{ - pallet::command::{ComponentRange, PovEstimationMode, PovModesMap}, + pallet::{ + command::{PovEstimationMode, PovModesMap}, + types::{ComponentRange, ComponentRangeMap}, + }, shared::UnderscoreHelper, PalletCmd, }; @@ -132,7 +135,7 @@ fn io_error(s: &str) -> std::io::Error { fn map_results( batches: &[BenchmarkBatchSplitResults], storage_info: &[StorageInfo], - component_ranges: &HashMap<(Vec, Vec), Vec>, + component_ranges: &ComponentRangeMap, pov_modes: PovModesMap, default_pov_mode: PovEstimationMode, analysis_choice: &AnalysisChoice, @@ -188,7 +191,7 @@ fn get_benchmark_data( batch: &BenchmarkBatchSplitResults, storage_info: &[StorageInfo], // Per extrinsic component ranges. - component_ranges: &HashMap<(Vec, Vec), Vec>, + component_ranges: &ComponentRangeMap, pov_modes: PovModesMap, default_pov_mode: PovEstimationMode, analysis_choice: &AnalysisChoice, @@ -207,6 +210,8 @@ fn get_benchmark_data( AnalysisChoice::MedianSlopes => Analysis::median_slopes, AnalysisChoice::Max => Analysis::max, }; + let pallet = String::from_utf8(batch.pallet.clone()).unwrap(); + let benchmark = String::from_utf8(batch.benchmark.clone()).unwrap(); let extrinsic_time = analysis_function(&batch.time_results, BenchmarkSelector::ExtrinsicTime) .expect("analysis function should return an extrinsic time for valid inputs"); @@ -282,10 +287,7 @@ fn get_benchmark_data( // We add additional comments showing which storage items were touched. // We find the worst case proof size, and use that as the final proof size result. let mut storage_per_prefix = HashMap::, Vec>::new(); - let pov_mode = pov_modes - .get(&(batch.pallet.clone(), batch.benchmark.clone())) - .cloned() - .unwrap_or_default(); + let pov_mode = pov_modes.get(&(pallet.clone(), benchmark.clone())).cloned().unwrap_or_default(); let comments = process_storage_results( &mut storage_per_prefix, &batch.db_results, @@ -351,12 +353,12 @@ fn get_benchmark_data( .collect::>(); let component_ranges = component_ranges - .get(&(batch.pallet.clone(), batch.benchmark.clone())) + .get(&(pallet.clone(), benchmark.clone())) .map(|c| c.clone()) .unwrap_or_default(); BenchmarkData { - name: String::from_utf8(batch.benchmark.clone()).unwrap(), + name: benchmark, components, base_weight: extrinsic_time.base, base_reads: reads.base, @@ -378,7 +380,7 @@ fn get_benchmark_data( pub(crate) fn write_results( batches: &[BenchmarkBatchSplitResults], storage_info: &[StorageInfo], - component_ranges: &HashMap<(Vec, Vec), Vec>, + component_ranges: &HashMap<(String, String), Vec>, pov_modes: PovModesMap, default_pov_mode: PovEstimationMode, path: &PathBuf, @@ -871,10 +873,10 @@ mod test { fn test_pov_mode() -> PovModesMap { let mut map = PovModesMap::new(); - map.entry((b"scheduler".to_vec(), b"first_benchmark".to_vec())) + map.entry(("scheduler".into(), "first_benchmark".into())) .or_default() .insert(("scheduler".into(), "mel".into()), PovEstimationMode::MaxEncodedLen); - map.entry((b"scheduler".to_vec(), b"first_benchmark".to_vec())) + map.entry(("scheduler".into(), "first_benchmark".into())) .or_default() .insert(("scheduler".into(), "measured".into()), PovEstimationMode::Measured); map diff --git a/substrate/utils/frame/omni-bencher/Cargo.toml b/substrate/utils/frame/omni-bencher/Cargo.toml new file mode 100644 index 000000000000..0c2d1a1b32b1 --- /dev/null +++ b/substrate/utils/frame/omni-bencher/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "frame-omni-bencher" +version = "0.1.0" +description = "Freestanding benchmark runner for any Polkadot runtime." +authors.workspace = true +edition.workspace = true +repository.workspace = true +license.workspace = true + +[lints] +workspace = true + +[dependencies] +clap = { version = "4.5.2", features = ["derive"] } +cumulus-primitives-proof-size-hostfunction = { path = "../../../../cumulus/primitives/proof-size-hostfunction" } +frame-benchmarking-cli = { path = "../benchmarking-cli", default-features = false } +sc-cli = { path = "../../../client/cli" } +sp-runtime = { path = "../../../primitives/runtime" } +sp-statement-store = { path = "../../../primitives/statement-store" } +env_logger = "0.11.2" +log = { workspace = true } diff --git a/substrate/utils/frame/omni-bencher/src/command.rs b/substrate/utils/frame/omni-bencher/src/command.rs new file mode 100644 index 000000000000..f0159f4307d6 --- /dev/null +++ b/substrate/utils/frame/omni-bencher/src/command.rs @@ -0,0 +1,149 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use clap::Parser; +use frame_benchmarking_cli::BenchmarkCmd; +use sc_cli::Result; +use sp_runtime::traits::BlakeTwo256; + +/// # Polkadot Omni Benchmarking CLI +/// +/// The Polkadot Omni benchmarker allows to benchmark the extrinsics of any Polkadot runtime. It is +/// meant to replace the current manual integration of the `benchmark pallet` into every parachain +/// node. This reduces duplicate code and makes maintenance for builders easier. The CLI is +/// currently only able to benchmark extrinsics. In the future it is planned to extend this to some +/// other areas. +/// +/// General FRAME runtimes could also be used with this benchmarker, as long as they don't utilize +/// any host functions that are not part of the Polkadot host specification. +/// +/// ## Installation +/// +/// Directly via crates.io: +/// +/// ```sh +/// cargo install --locked frame-omni-bencher +/// ``` +/// +/// or when the sources are locally checked out: +/// +/// ```sh +/// cargo install --locked --path substrate/utils/frame/omni-bencher --profile=production +/// ``` +/// +/// Check the installed version and print the docs: +/// +/// ```sh +/// frame-omni-bencher --help +/// ``` +/// +/// ## Usage +/// +/// First we need to ensure that there is a runtime available. As example we will build the Westend +/// runtime: +/// +/// ```sh +/// cargo build -p westend-runtime --profile production --features runtime-benchmarks +/// ``` +/// +/// Now as example we benchmark `pallet_balances`: +/// +/// ```sh +/// frame-omni-bencher v1 benchmark pallet \ +/// --runtime target/release/wbuild/westend-runtime/westend-runtime.compact.compressed.wasm \ +/// --pallet "pallet_balances" --extrinsic "" +/// ``` +/// +/// For the exact arguments of the `pallet` command, please refer to the `pallet` sub-module. +/// +/// ## Backwards Compatibility +/// +/// The exposed pallet sub-command is identical as the node-integrated CLI. The only difference is +/// that it needs to be prefixed with a `v1` to ensure drop-in compatibility. +#[derive(Parser, Debug)] +#[clap(author, version, about, verbatim_doc_comment)] +pub struct Command { + #[command(subcommand)] + sub: SubCommand, +} + +/// Root-level subcommands. +#[derive(Debug, clap::Subcommand)] +pub enum SubCommand { + /// Compatibility syntax with the old benchmark runner. + V1(V1Command), + // NOTE: Here we can add new commands in a forward-compatible way. For example when + // transforming the CLI from a monolithic design to a data driven pipeline, there could be + // commands like `measure`, `analyze` and `render`. +} + +/// A command that conforms to the legacy `benchmark` argument syntax. +#[derive(Parser, Debug)] +pub struct V1Command { + #[command(subcommand)] + sub: V1SubCommand, +} + +/// The `v1 benchmark` subcommand. +#[derive(Debug, clap::Subcommand)] +pub enum V1SubCommand { + Benchmark(V1BenchmarkCommand), +} + +/// Subcommands for `v1 benchmark`. +#[derive(Parser, Debug)] +pub struct V1BenchmarkCommand { + #[command(subcommand)] + sub: BenchmarkCmd, +} + +type HostFunctions = ( + sp_statement_store::runtime_api::HostFunctions, + cumulus_primitives_proof_size_hostfunction::storage_proof_size::HostFunctions, +); + +impl Command { + pub fn run(self) -> Result<()> { + match self.sub { + SubCommand::V1(V1Command { sub }) => sub.run(), + } + } +} + +impl V1SubCommand { + pub fn run(self) -> Result<()> { + let pallet = match self { + V1SubCommand::Benchmark(V1BenchmarkCommand { sub }) => match sub { + BenchmarkCmd::Pallet(pallet) => pallet, + _ => + return Err( + "Only the `v1 benchmark pallet` command is currently supported".into() + ), + }, + }; + + if let Some(spec) = pallet.shared_params.chain { + return Err(format!( + "Chain specs are not supported. Please remove `--chain={spec}` and use \ + `--runtime=` instead" + ) + .into()) + } + + pallet.run_with_spec::(None) + } +} diff --git a/substrate/utils/frame/omni-bencher/src/main.rs b/substrate/utils/frame/omni-bencher/src/main.rs new file mode 100644 index 000000000000..c148403f2970 --- /dev/null +++ b/substrate/utils/frame/omni-bencher/src/main.rs @@ -0,0 +1,29 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod command; + +use clap::Parser; +use env_logger::Env; +use sc_cli::Result; + +fn main() -> Result<()> { + env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); + log::warn!("The FRAME omni-bencher is not yet battle tested - double check the results.",); + + command::Command::parse().run() +} diff --git a/templates/parachain/node/src/command.rs b/templates/parachain/node/src/command.rs index 82624ae0be59..4c36fae30a26 100644 --- a/templates/parachain/node/src/command.rs +++ b/templates/parachain/node/src/command.rs @@ -184,7 +184,7 @@ pub fn run() -> Result<()> { match cmd { BenchmarkCmd::Pallet(cmd) => if cfg!(feature = "runtime-benchmarks") { - runner.sync_run(|config| cmd.run::, ReclaimHostFunctions>(config)) + runner.sync_run(|config| cmd.run_with_spec::, ReclaimHostFunctions>(Some(config.chain_spec))) } else { Err("Benchmarking wasn't enabled when building the node. \ You can enable it with `--features runtime-benchmarks`." diff --git a/templates/solochain/node/src/command.rs b/templates/solochain/node/src/command.rs index 42d1477f22f1..1b831f9cbbfd 100644 --- a/templates/solochain/node/src/command.rs +++ b/templates/solochain/node/src/command.rs @@ -117,7 +117,9 @@ pub fn run() -> sc_cli::Result<()> { ) } - cmd.run::, ()>(config) + cmd.run_with_spec::, ()>(Some( + config.chain_spec, + )) }, BenchmarkCmd::Block(cmd) => { let PartialComponents { client, .. } = service::new_partial(&config)?;