From 4c0abc5253960c67ff19ed67f3c9d191b4b70870 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 3 Feb 2021 16:25:02 +0000 Subject: [PATCH 01/18] A clean new attempt --- .gitignore | 1 + Cargo.lock | 141 ++++- Cargo.toml | 3 + bin/node/cli/Cargo.toml | 3 + bin/node/cli/src/cli.rs | 3 + bin/node/cli/src/command.rs | 7 + bin/node/runtime/Cargo.toml | 5 + bin/node/runtime/src/lib.rs | 39 +- client/state-db/src/lib.rs | 11 +- frame/executive/src/lib.rs | 41 +- frame/support/src/traits.rs | 32 +- primitives/state-machine/src/testing.rs | 18 +- primitives/storage/src/lib.rs | 9 +- .../dry-run-runtime-upgrade/api/Cargo.toml | 14 + .../dry-run-runtime-upgrade/api/src/lib.rs | 10 + .../dry-run-runtime-upgrade/cli/Cargo.toml | 24 + .../dry-run-runtime-upgrade/cli/src/lib.rs | 140 +++++ .../remote-externalities/.gitignore | 1 + .../remote-externalities/Cargo.toml | 26 + .../remote-externalities/proxy_test | Bin 0 -> 26476 bytes .../remote-externalities/src/lib.rs | 482 ++++++++++++++++++ 21 files changed, 971 insertions(+), 39 deletions(-) create mode 100644 utils/frame/dry-run-runtime-upgrade/api/Cargo.toml create mode 100644 utils/frame/dry-run-runtime-upgrade/api/src/lib.rs create mode 100644 utils/frame/dry-run-runtime-upgrade/cli/Cargo.toml create mode 100644 utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs create mode 100644 utils/frame/dry-run-runtime-upgrade/remote-externalities/.gitignore create mode 100644 utils/frame/dry-run-runtime-upgrade/remote-externalities/Cargo.toml create mode 100644 utils/frame/dry-run-runtime-upgrade/remote-externalities/proxy_test create mode 100644 utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs diff --git a/.gitignore b/.gitignore index c8f1ea9567bc2..ce302c74e10a0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ rls*.log **/hfuzz_workspace/ .cargo/ .cargo-remote.toml +*.bin diff --git a/Cargo.lock b/Cargo.lock index 7340f35ca0d04..50a7c41ae3c64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -192,6 +192,16 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "695579f0f2520f3774bb40461e5adb066459d4e0af4d59d20175484fb8e9edf1" +[[package]] +name = "async-attributes" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "async-channel" version = "1.5.1" @@ -219,12 +229,15 @@ dependencies = [ [[package]] name = "async-global-executor" -version = "1.4.3" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73079b49cd26b8fd5a15f68fc7707fc78698dc2a3d61430f2a7a9430230dfa04" +checksum = "9586ec52317f36de58453159d48351bc244bc24ced3effc1fce22f3d48664af6" dependencies = [ + "async-channel", "async-executor", "async-io", + "async-mutex", + "blocking", "futures-lite", "num_cpus", "once_cell", @@ -250,6 +263,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "async-lock" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1996609732bde4a9988bc42125f55f2af5f3c36370e27c778d5191a4a1b63bfb" +dependencies = [ + "event-listener", +] + [[package]] name = "async-mutex" version = "1.4.0" @@ -259,16 +281,34 @@ dependencies = [ "event-listener", ] +[[package]] +name = "async-process" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8cea09c1fb10a317d1b5af8024eeba256d6554763e85ecd90ff8df31c7bbda" +dependencies = [ + "async-io", + "blocking", + "cfg-if 0.1.10", + "event-listener", + "futures-lite", + "once_cell", + "signal-hook", + "winapi 0.3.9", +] + [[package]] name = "async-std" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7e82538bc65a25dbdff70e4c5439d52f068048ab97cdea0acd73f131594caa1" +checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341" dependencies = [ + "async-attributes", + "async-channel", "async-global-executor", "async-io", - "async-mutex", - "blocking", + "async-lock", + "async-process", "crossbeam-utils 0.8.0", "futures-channel", "futures-core", @@ -280,7 +320,7 @@ dependencies = [ "memchr", "num_cpus", "once_cell", - "pin-project-lite 0.1.11", + "pin-project-lite 0.2.0", "pin-utils", "slab", "wasm-bindgen-futures", @@ -1313,6 +1353,32 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dry-run-runtime-upgrade-api" +version = "0.1.0" +dependencies = [ + "sp-api", +] + +[[package]] +name = "dry-run-runtime-upgrade-cli" +version = "2.0.0" +dependencies = [ + "dry-run-runtime-upgrade-api", + "remote-externalities", + "sc-cli", + "sc-client-api", + "sc-executor", + "sc-service", + "sp-api", + "sp-blockchain", + "sp-core", + "sp-externalities", + "sp-runtime", + "sp-state-machine", + "structopt", +] + [[package]] name = "dyn-clonable" version = "0.9.0" @@ -1396,7 +1462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" dependencies = [ "atty", - "humantime", + "humantime 1.3.0", "log", "regex", "termcolor", @@ -1409,7 +1475,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", - "humantime", + "humantime 1.3.0", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "env_logger" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +dependencies = [ + "atty", + "humantime 2.1.0", "log", "regex", "termcolor", @@ -2362,6 +2441,12 @@ dependencies = [ "quick-error 1.2.3", ] +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.12.35" @@ -3826,6 +3911,7 @@ name = "node-cli" version = "2.0.0" dependencies = [ "assert_cmd", + "dry-run-runtime-upgrade-cli", "frame-benchmarking-cli", "frame-support", "frame-system", @@ -4009,6 +4095,7 @@ dependencies = [ name = "node-runtime" version = "2.0.1" dependencies = [ + "dry-run-runtime-upgrade-api", "frame-benchmarking", "frame-executive", "frame-support", @@ -6298,6 +6385,24 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "remote-externalities" +version = "0.1.0" +dependencies = [ + "async-std", + "bincode", + "env_logger 0.8.2", + "futures 0.1.30", + "hex-literal", + "jsonrpc-core-client", + "log", + "sc-rpc", + "sc-rpc-api", + "sp-core", + "sp-io", + "tokio 0.1.22", +] + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -7961,6 +8066,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" +[[package]] +name = "signal-hook" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e31d442c16f047a671b5a71e2161d6e68814012b7f5379d269ebd915fac2729" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.2.2" @@ -8882,9 +8997,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "structopt" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126d630294ec449fae0b16f964e35bf3c74f940da9dca17ee9b905f7b3112eb8" +checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" dependencies = [ "clap", "lazy_static", @@ -8893,9 +9008,9 @@ dependencies = [ [[package]] name = "structopt-derive" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e51c492f9e23a220534971ff5afc14037289de430e3c83f9daf6a1b6ae91e8" +checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" dependencies = [ "heck", "proc-macro-error", diff --git a/Cargo.toml b/Cargo.toml index 38b3a2bdcf296..14029293ab350 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -186,6 +186,9 @@ members = [ "utils/fork-tree", "utils/frame/benchmarking-cli", "utils/frame/frame-utilities-cli", + "utils/frame/dry-run-runtime-upgrade/remote-externalities", + "utils/frame/dry-run-runtime-upgrade/cli", + "utils/frame/dry-run-runtime-upgrade/api", "utils/frame/rpc/support", "utils/frame/rpc/system", "utils/prometheus", diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index af27b52377a43..79da5e61ed4b7 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -99,6 +99,7 @@ node-executor = { version = "2.0.0", path = "../executor" } sc-cli = { version = "0.8.0", optional = true, path = "../../../client/cli" } frame-benchmarking-cli = { version = "2.0.0", optional = true, path = "../../../utils/frame/benchmarking-cli" } node-inspect = { version = "0.8.0", optional = true, path = "../inspect" } +dry-run-runtime-upgrade-cli = { version = "2.0.0", optional = true, path = "../../../utils/frame/dry-run-runtime-upgrade/cli" } # WASM-specific dependencies wasm-bindgen = { version = "0.2.57", optional = true } @@ -131,6 +132,7 @@ node-inspect = { version = "0.8.0", optional = true, path = "../inspect" } frame-benchmarking-cli = { version = "2.0.0", optional = true, path = "../../../utils/frame/benchmarking-cli" } substrate-build-script-utils = { version = "2.0.0", optional = true, path = "../../../utils/build-script-utils" } substrate-frame-cli = { version = "2.0.0", optional = true, path = "../../../utils/frame/frame-utilities-cli" } +dry-run-runtime-upgrade-cli = { version = "2.0.0", optional = true, path = "../../../utils/frame/dry-run-runtime-upgrade/cli" } [build-dependencies.sc-cli] version = "0.8.0" @@ -155,6 +157,7 @@ cli = [ "sc-finality-grandpa-warp-sync", "structopt", "substrate-build-script-utils", + "dry-run-runtime-upgrade-cli", ] runtime-benchmarks = [ "node-runtime/runtime-benchmarks", diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 63a07e00e2197..6fcc475431076 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -47,6 +47,9 @@ pub enum Subcommand { #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), + /// DryRun all of the runtime upgrade hooks in the current runtime upon a configurable state. + DryRunRuntimeUpgrade(dry_run_runtime_upgrade_cli::DryRunCmd), + /// Verify a signature for a message, provided on STDIN, with a given (public or secret) key. Verify(VerifyCmd), diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 461930a613d97..297de09a08a2a 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -149,5 +149,12 @@ pub fn run() -> Result<()> { Ok((cmd.run(client, backend), task_manager)) }) }, + Some(Subcommand::DryRun(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { task_manager, .. } = new_partial(&config)?; + Ok((cmd.run::(config), task_manager)) + }) + } } } diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index d6b38802fe691..55f875d0a7773 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -83,6 +83,10 @@ pallet-transaction-payment = { version = "2.0.0", default-features = false, path pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } pallet-vesting = { version = "2.0.0", default-features = false, path = "../../../frame/vesting" } +dry-run-runtime-upgrade-api = { default-features = false, path = "../../../utils/frame/dry-run-runtime-upgrade/api" } +# TODO: probably best to put this in some primitive pallet, among with types that need to be passed +# back and forth. + [build-dependencies] substrate-wasm-builder = { version = "3.0.0", path = "../../../utils/wasm-builder" } @@ -151,6 +155,7 @@ std = [ "pallet-society/std", "pallet-recovery/std", "pallet-vesting/std", + "dry-run-runtime-upgrade-api/std", ] runtime-benchmarks = [ "frame-benchmarking", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index c2a2542b5588a..a2e541ba7d80e 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1078,7 +1078,14 @@ pub type SignedPayload = generic::SignedPayload; /// Extrinsic type that has already been checked. pub type CheckedExtrinsic = generic::CheckedExtrinsic; /// Executive: handles dispatch to the various modules. -pub type Executive = frame_executive::Executive, Runtime, AllModules>; +pub type Executive = frame_executive::Executive< + Runtime, + Block, + frame_system::ChainContext, + Runtime, + AllModules, + (), +>; /// MMR helper types. mod mmr { @@ -1323,15 +1330,39 @@ impl_runtime_apis! { } } + // TODO: make everything feature gated. + impl dry_run_runtime_upgrade_api::DryRunRuntimeUpgrade for Runtime { + fn dry_run_runtime_upgrade() -> Weight { + frame_support::debug::RuntimeLogger::init(); + frame_support::debug::info!("!!! DRYRUN MIGRATION UP AHEAD !!!"); + let weight = Executive::dry_run_runtime_upgrade(); + frame_support::debug::info!("!!! DRYRUN MIGRATION DONE !!!"); + weight + + // \migration_bot pallet:staking state:polkadot + // pseudo code: + // match config.pallet { + // "System" => { + // ::pre_migration(); + // ::on_runtime_upgrade(); + // ::post_migration(); + // }, + // "All" => { + // Executive::dry_run_runtime_upgrade() + // } + // } + } + } + #[cfg(feature = "runtime-benchmarks")] impl frame_benchmarking::Benchmark for Runtime { fn dispatch_benchmark( config: frame_benchmarking::BenchmarkConfig ) -> Result, sp_runtime::RuntimeString> { use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey}; - // Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues. - // To get around that, we separated the Session benchmarks into its own crate, which is why - // we need these two lines below. + // Trying to add benchmarks directly to the Session Pallet caused cyclic dependency + // issues. To get around that, we separated the Session benchmarks into its own crate, + // which is why we need these two lines below. use pallet_session_benchmarking::Module as SessionBench; use pallet_offences_benchmarking::Module as OffencesBench; use frame_system_benchmarking::Module as SystemBench; diff --git a/client/state-db/src/lib.rs b/client/state-db/src/lib.rs index dd2baf9d18ace..a8a6bd7bc9437 100644 --- a/client/state-db/src/lib.rs +++ b/client/state-db/src/lib.rs @@ -24,12 +24,12 @@ //! Canonicalization window tracks a tree of blocks identified by header hash. The in-memory //! overlay allows to get any node that was inserted in any of the blocks within the window. //! The tree is journaled to the backing database and rebuilt on startup. -//! Canonicalization function selects one root from the top of the tree and discards all other roots and -//! their subtrees. +//! Canonicalization function selects one root from the top of the tree and discards all other roots +//! and their subtrees. //! //! # Pruning. -//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until pruning -//! constraints are satisfied. +//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until +//! pruning constraints are satisfied. mod noncanonical; mod pruning; @@ -237,8 +237,7 @@ impl StateDbSync Ok(()), - Some(v) => Err(Error::InvalidPruningMode(String::from_utf8_lossy(v).into())), - None => Ok(()), + _ => Ok(()), } } diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index df1ae17df613f..b7a0f91282ce4 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -44,7 +44,8 @@ //! //! ## Usage //! -//! The default Substrate node template declares the [`Executive`](./struct.Executive.html) type in its library. +//! The default Substrate node template declares the [`Executive`](./struct.Executive.html) type in +//! its library. //! //! ### Example //! @@ -205,6 +206,39 @@ where OriginOf: From>, UnsignedValidator: ValidateUnsigned>, { + /// Execute all `OnRuntimeUpgrade` of this runtime, and return the aggregate weight. + pub fn do_on_runtime_upgrade() -> frame_support::weights::Weight { + let mut weight = 0; + weight = weight.saturating_add( + as OnRuntimeUpgrade>::on_runtime_upgrade(), + ); + weight = weight.saturating_add(COnRuntimeUpgrade::on_runtime_upgrade()); + weight = weight.saturating_add(::on_runtime_upgrade()); + + weight + } + + /// Execute all `OnRuntimeUpgrade` of this runtime, including the pre and post migration checks. + /// + /// This should only be used for testing. + pub fn dry_run_runtime_upgrade() -> frame_support::weights::Weight { + < + (frame_system::Module::, COnRuntimeUpgrade, AllModules) + as + OnRuntimeUpgrade + >::pre_migration().expect("pre-migration hook failed."); + + let weight = Self::do_on_runtime_upgrade(); + + < + (frame_system::Module::, COnRuntimeUpgrade, AllModules) + as + OnRuntimeUpgrade + >::post_migration().expect("post-migration hook failed."); + + weight + } + /// Start the execution of a particular block. pub fn initialize_block(header: &System::Header) { sp_io::init_tracing(); @@ -234,10 +268,7 @@ where ) { let mut weight = 0; if Self::runtime_upgraded() { - // System is not part of `AllModules`, so we need to call this manually. - weight = weight.saturating_add( as OnRuntimeUpgrade>::on_runtime_upgrade()); - weight = weight.saturating_add(COnRuntimeUpgrade::on_runtime_upgrade()); - weight = weight.saturating_add(::on_runtime_upgrade()); + weight = weight.saturating_add(Self::do_on_runtime_upgrade()); } >::initialize( block_number, diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index c52aa60c20b15..0fbb7a80b798c 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -1552,7 +1552,25 @@ pub trait OnRuntimeUpgrade { /// block local data are not accessible. /// /// Return the non-negotiable weight consumed for runtime upgrade. - fn on_runtime_upgrade() -> crate::weights::Weight { 0 } + fn on_runtime_upgrade() -> crate::weights::Weight { + 0 + } + + /// Execute some pre-checks prior to a runtime upgrade. + /// + /// These hooks are never meant to be executed on-chain, instead only be used by 3rd party tools + /// for testing. + fn pre_migration() -> Result<(), &'static str> { + Ok(()) + } + + /// Execute some post-checks prior to a runtime upgrade. + /// + /// These hooks are never meant to be executed on-chain, instead only be used by 3rd party tools + /// for testing. + fn post_migration() -> Result<(), &'static str> { + Ok(()) + } } #[impl_for_tuples(30)] @@ -1562,6 +1580,18 @@ impl OnRuntimeUpgrade for Tuple { for_tuples!( #( weight = weight.saturating_add(Tuple::on_runtime_upgrade()); )* ); weight } + + fn pre_migration() -> Result<(), &'static str> { + let mut result = Ok(()); + for_tuples!( #( result = result.and(Tuple::pre_migration()); )* ); + result + } + + fn post_migration() -> Result<(), &'static str> { + let mut result = Ok(()); + for_tuples!( #( result = result.and(Tuple::post_migration()); )* ); + result + } } /// Off-chain computation trait. diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index a6f9d06824642..6ec27fc0a8d12 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -48,20 +48,22 @@ pub struct TestExternalities where H::Out: codec::Codec + Ord, { - overlay: OverlayedChanges, + /// The overlay changed storage. + pub overlay: OverlayedChanges, offchain_db: TestPersistentOffchainDB, - storage_transaction_cache: StorageTransactionCache< - as Backend>::Transaction, H, N - >, - backend: InMemoryBackend, + storage_transaction_cache: + StorageTransactionCache< as Backend>::Transaction, H, N>, + /// Storage backend. + pub backend: InMemoryBackend, changes_trie_config: Option, changes_trie_storage: ChangesTrieInMemoryStorage, - extensions: Extensions, + /// Extensions. + pub extensions: Extensions, } impl TestExternalities - where - H::Out: Ord + 'static + codec::Codec +where + H::Out: Ord + 'static + codec::Codec, { /// Get externalities implementation. pub fn ext(&mut self) -> Ext> { diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index 1e9f9766072e4..1016b73eb1e3b 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -31,10 +31,15 @@ use codec::{Encode, Decode}; #[derive(PartialEq, Eq, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))] pub struct StorageKey( - #[cfg_attr(feature = "std", serde(with="impl_serde::serialize"))] - pub Vec, + #[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))] pub Vec, ); +impl AsRef<[u8]> for StorageKey { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + /// Storage key with read/write tracking information. #[derive(PartialEq, Eq, RuntimeDebug, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Hash, PartialOrd, Ord))] diff --git a/utils/frame/dry-run-runtime-upgrade/api/Cargo.toml b/utils/frame/dry-run-runtime-upgrade/api/Cargo.toml new file mode 100644 index 0000000000000..b4eafa6c40e19 --- /dev/null +++ b/utils/frame/dry-run-runtime-upgrade/api/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "dry-run-runtime-upgrade-api" +version = "0.1.0" +authors = ["kianenigma "] +edition = "2018" + +[dependencies] +sp-api = { version = "2.0.1", path = "../../../../primitives/api", default-features = false } + +[features] +default = [ "std" ] +std = [ + "sp-api/std", +] diff --git a/utils/frame/dry-run-runtime-upgrade/api/src/lib.rs b/utils/frame/dry-run-runtime-upgrade/api/src/lib.rs new file mode 100644 index 0000000000000..cd53ca97f175a --- /dev/null +++ b/utils/frame/dry-run-runtime-upgrade/api/src/lib.rs @@ -0,0 +1,10 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +sp_api::decl_runtime_apis! { + /// Runtime api for testing the migration of a FRAME runtime. + // TODO: move this to frame. + pub trait DryRunRuntimeUpgrade { + /// dry-run runtime upgrades, returning the total weight consumed. + fn dry_run_runtime_upgrade() -> u64; + } +} diff --git a/utils/frame/dry-run-runtime-upgrade/cli/Cargo.toml b/utils/frame/dry-run-runtime-upgrade/cli/Cargo.toml new file mode 100644 index 0000000000000..48e7c9d70066a --- /dev/null +++ b/utils/frame/dry-run-runtime-upgrade/cli/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "dry-run-runtime-upgrade-cli" +version = "2.0.0" +authors = ["kianenigma "] +edition = "2018" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +sc-service = { version = "0.8.0", default-features = false, path = "../../../../client/service" } +sc-cli = { version = "0.8.0", path = "../../../../client/cli" } +sc-executor = { path = "../../../../client/executor" } +sc-client-api = { version = "2.0.0", path = "../../../../client/api" } +structopt = "0.3.8" +sp-state-machine = { version = "0.8.1", path = "../../../../primitives/state-machine" } +sp-api = { version = "2.0.1", path = "../../../../primitives/api" } +sp-blockchain = { version = "2.0.1", path = "../../../../primitives/blockchain" } +sp-runtime = { version = "2.0.1", path = "../../../../primitives/runtime" } +sp-externalities = { path = "../../../../primitives/externalities" } +sp-core = { path = "../../../../primitives/core" } + +dry-run-runtime-upgrade-api = { path = "../api" } +remote-externalities = { path = "../remote-externalities" } diff --git a/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs b/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs new file mode 100644 index 0000000000000..02ebe33bb9767 --- /dev/null +++ b/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs @@ -0,0 +1,140 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{fmt::Debug, str::FromStr}; +use sc_service::Configuration; +use sc_cli::{CliConfiguration, ExecutionStrategy}; +use sc_executor::{WasmExecutionMethod, NativeExecutor}; +use sp_state_machine::StateMachine; +use sc_service::NativeExecutionDispatch; +use sp_runtime::traits::{Block as BlockT, NumberFor}; +use sp_core::storage::{StorageData, StorageKey, well_known_keys}; + +#[derive(Debug, structopt::StructOpt)] +pub struct DryRunCmd { + /// The shared parameters + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: sc_cli::SharedParams, + + /// The target pallet to run the migration against. + #[structopt(short, long, default_value = "All")] + pub target: Target, + + /// The state to use to run the migration. It should be a ":" separated string, where the + /// prefix is either `live` or `snap`, and the postfix is either the HTTP uri or the file path, + /// respectively. + #[structopt(short, long, default_value = "live:http://localhost:9933")] + pub state: State, +} + +/// Possible targets for dryrun runtime upgrade. +#[derive(Debug)] +pub enum Target { + /// All pallets. + All, + /// A single pallet. + Pallet(String), +} + +impl FromStr for Target { + type Err = &'static str; + fn from_str(s: &str) -> Result { + Ok(if s.to_lowercase() == "All" { Target::All } else { Target::Pallet(s.to_string()) }) + } +} + +/// The state to use for a migration dryrun. +#[derive(Debug)] +pub enum State { + /// A snapshot. Inner value is file path. + Snap(String), + + /// A live chain. Inner value is the HTTP uri. + Live(String), +} + +impl FromStr for State { + type Err = &'static str; + fn from_str(s: &str) -> Result { + let splitted = s.splitn(2, ":").collect::>(); + if splitted.len() != 2 { + return Err("invalid format. Must be [state_type]:[state_value]."); + } + let state_type = splitted[0]; + let value = splitted[1]; + match state_type { + "live" => Ok(State::Live(value.to_string())), + "snap" => Ok(State::Snap(value.to_string())), + _ => Err("Invalid state type."), + } + } +} + +impl DryRunCmd { + pub async fn run(&self, config: Configuration) -> sc_cli::Result<()> + where + B: BlockT, + ExecDispatch: NativeExecutionDispatch + 'static, + { + let spec = config.chain_spec; + let genesis_storage = spec.build_storage()?; + + let code = StorageData(genesis_storage.top.get(well_known_keys::CODE).unwrap().to_vec()); + let code_key = StorageKey(well_known_keys::CODE.to_vec()); + + let wasm_method = WasmExecutionMethod::Interpreted; + let strategy = ExecutionStrategy::Native; + let mut changes = Default::default(); + + let heap_pages = Some(1024); + let executor = NativeExecutor::::new(wasm_method, heap_pages, 2); + + let ext = remote_externalities::Builder::new().inject(&[(code_key, code)]).build().await; + + let raw_result = StateMachine::<_, _, NumberFor, _>::new( + &ext.backend, + None, + &mut changes, + &executor, + "DryRunRuntimeUpgrade_dry_run_runtime_upgrade", + &[], + ext.extensions, + &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) + .runtime_code() + .unwrap(), + sp_core::testing::TaskExecutor::new(), + ) + .execute(strategy.into()) + .unwrap(); + + Ok(()) + } +} + +impl CliConfiguration for DryRunCmd { + fn shared_params(&self) -> &sc_cli::SharedParams { + &self.shared_params + } + + fn chain_id(&self, _is_dev: bool) -> sc_cli::Result { + Ok(match self.shared_params.chain { + Some(ref chain) => chain.clone(), + None => "dev".into(), + }) + } +} diff --git a/utils/frame/dry-run-runtime-upgrade/remote-externalities/.gitignore b/utils/frame/dry-run-runtime-upgrade/remote-externalities/.gitignore new file mode 100644 index 0000000000000..a8a0dcec44720 --- /dev/null +++ b/utils/frame/dry-run-runtime-upgrade/remote-externalities/.gitignore @@ -0,0 +1 @@ +*.bin diff --git a/utils/frame/dry-run-runtime-upgrade/remote-externalities/Cargo.toml b/utils/frame/dry-run-runtime-upgrade/remote-externalities/Cargo.toml new file mode 100644 index 0000000000000..7128bfd3eeb35 --- /dev/null +++ b/utils/frame/dry-run-runtime-upgrade/remote-externalities/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "remote-externalities" +version = "0.1.0" +authors = ["kianenigma "] +edition = "2018" + +[dependencies] +jsonrpc-core-client = { version = "15.1.0", features = ["http"] } +sc-rpc-api = { path = "../../../../client/rpc-api" } +sc-rpc = { path = "../../../../client/rpc" } +futures = "0.1.29" + +hex-literal = "0.3.1" +env_logger = "0.8.2" +log = "0.4.11" +bincode = "1.3.1" +tokio = "0.1.22" + +sp-io = { path = "../../../../primitives/io" } +sp-core = { path = "../../../../primitives/core" } + +[dev-dependencies] +async-std = { version = "1.6.5", features = ["attributes"] } + +[features] +remote-test = [] diff --git a/utils/frame/dry-run-runtime-upgrade/remote-externalities/proxy_test b/utils/frame/dry-run-runtime-upgrade/remote-externalities/proxy_test new file mode 100644 index 0000000000000000000000000000000000000000..adb93f5ba270c3e3740394f183d96bc54a3211f9 GIT binary patch literal 26476 zcmd_zJC^Q9k{n=?;%M9h7k~)F15ihx#7zVu0Fm;ngR9~4_<45aOBwPov;V3(lQZ2_ zeQ#AJ{(y(OxtY1&|Nfu;@%6v{`0Ms>>DbQIkG-^>``GTM9&>xfwqN5Jd*0JJj=JV^ z<=&rk)F-dwZh82dbDKxq`gWC^(>(kuTiV9BeX~^4IBT6fUt?|i>iNmX)9#~etIhMM zXB$25eXKj3dwupgw$^K_?SFqf*>;qBUw51Jx{veP+B4eI%gFnE9HU;TweqBPUUxfd zyZy`Qe$K6zQt!3Zr`}uM{`jid??1)!ls%O*G@mEm z`OJISSDyLa_Uc#fR<3<1R^=!^EU!LoqejaDt_Gi1VanF^m$8(;?x>nmp zulMtKuygtCyz+C*EAKfuul3yPIlOJ3ZyU#RU452Oa!Fs$;}|I=f!~|ku;o10b&OK> zy{`K_i=VMp8mpdnD_bhZd7tGv_w&d#J!kQ2f5Le@dEU*t*=8%H?=3w~OG(6B=Uuk_ z+#cbbZb4LXzw0^gv7Kk%k37bC=H#GUmeO2R7x4Oe^RAIj!ElW0sQH-ZQ+JUssLIt+ zEop7ue^Dwlzgw*`?WXsy^iOU%I4X`L_Y3%)Y6_>q@=p9L)z=l zm1-|Xc}5jtDK$6mS}u*l-SgK-+e>luZ5``8gm1g|*{?Pn#-5In_pR-z_2wUY9+{~f zGMDI-i_POe_rJS}V+mGI=Ozy+?>vUZ6m=fdT|E}C_ zzw81%eI~hlS3CBRwy)>$)3g1IwAY=7va(-KIp)3Ac`ffR^&=-)rXK21>-{KdpaY30 zI@WWp{ke;ZGxix(XUf@qKhvd<-no3edB0clB8JvhRL(rkJ-03Gm%=HGxB1j`s2JtA z?`=rw+r9RyUqu*l>t?>{*Ze)ZF&gPld>n-yZXTQ}%s(j^S=JefBr|&+x~5vzjv~#Yd)W6Ynw_SFO=zpDgv|3*4!`jU5Bxd(`IOC|Ty&)&1R zntR_?5p<>8^F0@A+uRi4Gk)?vyAEfUe;jJ)waW1M>UjxnZ$^ZTTS0lwS#y*Rus^>;q!)b~s+(2U0HnH}gYPNt~wbLP6P-U53k1X)^b(JLg`zVs!T-14@giKEpfz{=zP7HL!W&;}ql zCh#2S=$3h+qbmV$%PtD_m~*?+@eEIJ$e$7yjB!Y(@23%<>Q%kg4yNn*<(*x7cu3c` zl*5u^HFY|={{4_$pd+`pq#0jQ~o zUzc|DoKPAD=)CYA`CCpa*t$#7%h-~Eo;tBFBD~9Tb#Jcr-L~tJJC&YGRc(sujd+2% z9M^`(c|VXB#_Vx6@cd|q&FwzZPnn6YIqj$1%V$oDVw;`|fBAW@-_I)k`g{ChId^GC zE-uGD)%546NVDX21|>0Xx%S?pZBh&6WR(qLs1^pd(e)Zzy7FlGK~W%YQvY^`tM32M zrs#X_XTm8-7IQPUOU2yLu-h#VBuv+4t5eOpP(K?wB*@Ay==-nOwY~5TkKi|D8-!WL zQ#UyDgvQGHM?FHxB8g6U>8yJf)3C4rKNnUQ-vA&_7ZjVNXn`2V*O)z`z0h@@K<=TU zMM{&MQ&Dj@EgBwR)8dX>Yjhr{W!X%0=>5`ErEwtdarwGCEHE&T<%BB%130sxpla`R z*YocM3yiMz$PyY_a-UQRy+c`7-kx)`mViycyl2?8UHu#LLXnWJ5n7-ya(5ungZe~p z9S0I;caS^675m>WMStKa^=i4tA>WX4a1t4j4&FU8=s#iy!ljWvrrsZop0F`428`4t zP~`dG~mdq3A#a-KEUsWRw!po|-|Afg=0=X#zx^-5ZgNI~evP!y6`&QpD;Ywdwzfjq@XC+_8Fjojr>sug)D{T? z3I-gd9x}|MyZJni7e<=r>Mj$qTiM5~yL{rv%fY`Cc^e#49fRw- zfrP{gw=}QO`u^=aTvLaL@j-Jtm_pERB(sX=+Y0e-^T{DWN=i5+H!aQ`r|*kcqGHu^GJub{;CQp zzIkdEwE8$_5O34mHsy2(LMK3dqTjf;TAqxbma!zjKnQ<)z3@ExBb98PMjzSy*Q!7a zSb$Hs9F11FOfjO+6pVOZyR>}5vdYQ1AHlvy{sMl6Cbf>QnMfDMl5hrFg^Mp+PA58b zNhKIXsr^P-tc-Lc6C*}MWxWWP#K08L=*xLH#(j;8`Wh%1mPiOSgpMcnPm(<~{k}Cs zC9AGrED!mURGTXPDX5e8uh(TFk4EFJI(P_uAPGgkt9WN6>B=I==UoYXui6cH6g{Jt z1Q?vsTl?SZ)cnQ24Bi~u*N)()c;d4|`aQ*P>OqRGMt0?bXfkR?RE9tvB1&38J}ckk zfdgi$%p@oLUXay;BZTd*mFTqIURJ7tk z#jDzfm}#mrLhwEIUx|Rj#e(*P$onL80}G*hOZ^Z(brI6jctH}kHi@fZJ-imR9~iIW zcy4OVP0KyV5Z+CJ_rK;@e8W1mVp!wJSQG!0I2w0P3J_OH4m%6&Rbn`Csa~HG(l@j% z?%Mp44PKXLJ8IYKn>i#X?`nH)#5WugtWH1>-4rrU52FFaTMn-qt8_a;G@s5O(Zp$K zVH4q$NMEzTFGkw0X|amPV7|yOhAa;X?9bUl3)cWLxzBWO3dYh+wKt5*8;|vPV z7{aHOuQB^~RLHt&Pg`MD*#?P>>x5S7*uujfR>&YI?o`Qqu|Jcn=YnHT^6iE z(xRex2AKP^dcH2F9ebXIF7fn?lfIUZyDLpB3tUYXOWUSEg?sMd=OA5hw0ZDdGd2Mo zNUWnkHS9xGz%qT)ae@?5j?r@!lcC&c%Z}n0H1?z5of#lPT1C1s5rU6_1T~= zH9iEIPF8`#BtQCCsyZr!5o+Z|0+59Y`k1qYNOUGy3S&TFBnf(==*mCP)eGZ;6LLQq zz6UquatDJ!vI9$4kNBba+bEKcCh>7?+5&PBc&sJ|QjnGf+TcVKaNyD9`o{V=GePgt zQS*4hG*(M_-A35ZO=Ru|g(m)nXHeb?D zeW73DQ(6NX+Bv?hm7@ZsUU1B~=Y1ln@D1K0?;WZeU0 zy!c#sn740ENI(O%Be&=`5f1?=UA@${?1m`U;j!S9m0kTZ*5dKKm=D5IfT<84A1dSP z;)4%b$hLr~$a54gJ4%cI%*;CLAe>P^LU@=fpQDu|%0Ut)HFgsDfQ5P^468G!Ecx|o ze4ykix?RsbBu~_*OU5cOM{2w$T8ZHF)LJ|(U>b@Q{2Y9Sp@1S*{|uJ&L^MoIyiW_Z zld^W6UYf8jjl~khUE0pxOp!%Z?0z{ENn#q}N|C7CB?Jm4LY{qn-ZlT~m>Ju|<9IX# zz`65CpaupT1fC?~{l?xdui?92dWFZkKmFb@P{WK9W)8Efe?gBd?pc* z*%4K2Tm$DQx`Lvfg5sh zItAx1Tf86b>o`xt8sfTKovy-3PF6wo0d&YTqMr+)+Ouw>*S3yHC)6T{58TEARCQHM z*<2HbIR@bDsK1`Ujk&i3S=EaB^)kk{icsEVu-u4L(fztGHfF8nkwA3Z2G0%glSlh~Ag> z?}c7|DmH{FD#C>m&AKi_{X)=+u1e2<;NJLGv_R(B4vY2-2oi84CvGXFaZO4MV7Scwj(w zQhsW_!mr5q$DQP#;2+>X*A*eiv`J-Yr-5U5ao_`tHn52@goa<8EhGS09y11oV+aVG zXP-uGQe9vYgRR)vzkW6d%?LTr5MUHDdbauL2R-yQ*BI*LeH(oCJNT1E%QoXjk;K$d z`?;?^GxqCH275lQ0ah?8C&pYA{2oqlswk?XFa!n(!b_onp$(@KgjFx~06q+ZSqvG3 z$p^`XhR)@Gd`w?y#(r(S;K$$T17-Z)q{5HC)5kM~pBhbwmi_EsKV#uH{h!9gkrpu9 zrBq?|F7sqZOqa_z^eQ#m-1HnV@rgL5B*;dI8H$-le+n&Db8kd;O%3XE?tk_AYn!C4K*bIDG-3XvKp4D6Wb0hFbiy1OO7#ae#c87V2OsCxDq^O`tcu|a5|A9k zQEIvfAzmnzn|=W&-D7kd=+!MFv+$o`qRRd*2A}^7ewGv((j0}@Fep_4_%J8w!y=(B z(*`Reo~l!M$o8xZY>8-EqqyM_%2iXdog|#gFe2?Xe>;ypeJWCkBuL}L(wSroD`ZTZ zJ!-UQkO$Gfi>O3!XXZpzgWXtBm&hQ`dw{%-e+vRyQ$Sb`uM%S{T}p={BElt0n1SxX z^p7)S0Wjf4v>7pcuZo)Vq5>il5pyC;7uLSj&I34%~0D7z|Doty1y$0YvmpYFZ6G!6+p?gcUV`EE03RfL1 z17&j^hBsJ=!~Mf8!2SPilfi#%_QO`25YeYI3XPpavl3fYGlpUv2p0pVM0uJIpxliFK}!0Y){Ku8WLYD-Wg2XMpnT5yM1; z^s#ir@iFWO5Hj(PzJLARMv&L_76t}jTJkhP(i`Af`Po*JZ+Z5vmZTkd3$KgJ8DnhB zF3^kNSv`%+tgYeGW}&uWO0uP?OUMRlC=u@9$JYz$db42 zII~c3%bzjj3S5T~y{|F*ciaJYV^WRD z7&I^(#-wCF1Q;4>q2W(uA7O`S)`3v$82G6v7!MFsAFaLD!|Uyh^>K)T@q+s6d2lHe zi8z3LEJ{q+0WidN0&XNrJ@k%}JW>**GkjKKpw0-qK3*&JmZ7XMiYjDXl#t2D4S*j3{pL@B7qru|L!b zWo?=W511t;rUq(tOLr~-qPhij{HF#Aa80JS1gj-zaf@|3VwA(S#`s&LCvfeDGvJaL zl!CI6L-hiTdEsd_^PoLsEz?8sGz1wK?92f!#e>HFTRD*ZIW+7)p5tfT1NH|K&ArZV zO%9HsYls28Yv3;C%WM%u7HLOfZYMTuFlktq=b3H#z&lOw^zA%;_+ISai?F2Fqgn0B zwS$M*k(h-#hk10`I%1uYac?rQbpZ^r=HyrqEdPv6gT$=gcrR~O^w|dG$$Td=%|xsw zHr7hS@)c7lTstvia)yF{_wpp^Z=XI0<|#J7+Zs}<4O=wtzPk? zn&&_!BAjRiXaSZ3_lQ+%Xtl!K1nA?ae|lexPb-e|7ISYFu~;n;8yeY@jCl(Ic1C^x zT&k_Gu_nQ!7w43CM#b|}F?am&96x(6+>^ZN3tgq5G@Of#bcd}Nj38*%DHJ>ZVvj|Y zOpI6>-p7gVM2K>LgB7+fU*w%PqZ|iRSKG}*r*1*0b z{2R{!;TqM;rVGphxgTIpEDMpP{Kqx5`Z~ml(98-&2jeH7y>5FND98yV?9>R6w%2QmFWda`5O^oZJNB%GHRb2$A!kMAieY1GvivA3 z5Ex~x(n2gf=jpv9F46L1jxW`yC#AKAyt|KZd zI|eIx2(Ai^H5}Y@0=Jt0Vt=*%5i$SV9i=Bcz?q81pk zG6@Dm2Qpe;*)`R%ffwEP8_!|wOSCOg0I(G%+?&C`#G+YDn|r7zNe|OodBi5;e_%Bx~Tbv-SqY(xe*;rq=Bq~P*5%p~e6I(L86fsI#wQPk=(6U0~ zgg<{Tx+de*Q7LR9W!V*~(wJeAx9nw78)xjuC3V#lIqk%Bj|Pr-Q3TdA=h)Dcufpcw)gL_hA&E%=4=2=OcS z0C*V|7LZ;glQ$CxW9{3j*SB3=G;Y#A3g@=f)56lQPduRu#v*^C=s$lis#Q$$$liuL zgkmF*K9X^-~NQGwJ0ez3@{3SK?VQ51R;Zod5~e>a@CbvVJM|mSVXZWdo2`iD(QKc|KQ3CH z9A%c<4}=ctolV1J0o_B@!@{dYTALU##rwua#$PxO=o1mDoVpKh`w4&^Ym;p*pj*XG zG(U^rLEMB-PWJb(TWa=VqCBKvT$(?j!Fv%S(7^waTdVZMD zt_m~8RLtAxf=q5+&XuO$#(*dTbqvmIv7_wg@d3){qW9SRwCvWAu}R37Kl$5ew2fUt p#^SFqX&9c6*Bw`EWr32VKMjsu$nhNTtv|y?__%zZ{QT$N{U5`qT;l)$ literal 0 HcmV?d00001 diff --git a/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs b/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs new file mode 100644 index 0000000000000..cefaddb2378e6 --- /dev/null +++ b/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs @@ -0,0 +1,482 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Remote Externalities +//! +//! An equivalent of `sp_io::TestExternalities` that can load its state from a remote substrate +//! based chain, or a local cache file. +//! +//! #### Runtime to Test Against +//! +//! While not absolutely necessary, you most likely need a `Runtime` equivalent in your test setup +//! through which you can infer storage types. There are two options here: +//! +//! 1. Build a mock runtime, similar how to you would build one in a pallet test (see example +//! below). The very important point here is that this mock needs to hold real values for types +//! that matter for you, based on the chain of interest. Some typical ones are: +//! +//! - `sp_runtime::AccountId32` as `AccountId`. +//! - `u32` as `BlockNumber`. +//! - `u128` as Balance. +//! +//! Once you have your `Runtime`, you can use it for storage type resolution and do things like +//! `>::storage_getter()` or `>::get()`. +//! +//! 2. Or, you can use a real runtime. +//! +//! ### Example +//! +//! With a test runtime +//! +//! ```ignore +//! use remote_externalities::Builder; +//! +//! #[derive(Clone, Eq, PartialEq, Debug, Default)] +//! pub struct TestRuntime; +//! +//! use frame_system as system; +//! impl_outer_origin! { +//! pub enum Origin for TestRuntime {} +//! } +//! +//! impl frame_system::Config for TestRuntime { +//! .. +//! // we only care about these two for now. The rest can be mock. The block number type of +//! // kusama is u32. +//! type BlockNumber = u32; +//! type Header = Header; +//! .. +//! } +//! +//! #[test] +//! fn test_runtime_works() { +//! let hash: Hash = +//! hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into(); +//! let parent: Hash = +//! hex!["540922e96a8fcaf945ed23c6f09c3e189bd88504ec945cc2171deaebeaf2f37e"].into(); +//! Builder::new() +//! .at(hash) +//! .module("System") +//! .build() +//! .execute_with(|| { +//! assert_eq!( +//! // note: the hash corresponds to 3098546. We can check only the parent. +//! // https://polkascan.io/kusama/block/3098546 +//! >::block_hash(3098545u32), +//! parent, +//! ) +//! }); +//! } +//! ``` +//! +//! Or with the real kusama runtime. +//! +//! ```ignore +//! use remote_externalities::Builder; +//! use kusama_runtime::Runtime; +//! +//! #[test] +//! fn test_runtime_works() { +//! let hash: Hash = +//! hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into(); +//! Builder::new() +//! .at(hash) +//! .module("Staking") +//! .build() +//! .execute_with(|| assert_eq!(>::validator_count(), 400)); +//! } +//! ``` + +use std::{ + fs, + path::{Path, PathBuf}, +}; +use std::fmt::{Debug, Formatter, Result as FmtResult}; +use log::*; +use sp_core::{hashing::twox_128}; +pub use sp_io::TestExternalities; +use sp_core::storage::{StorageKey, StorageData}; +use futures::future::Future; + +type KeyPair = (StorageKey, StorageData); +type Number = u32; +type Hash = sp_core::H256; +// TODO: make these two generic. + +const LOG_TARGET: &'static str = "remote-ext"; + +/// Struct for better hex printing of slice types. +pub struct HexSlice<'a>(&'a [u8]); + +impl<'a> HexSlice<'a> { + pub fn new(data: &'a T) -> HexSlice<'a> + where + T: ?Sized + AsRef<[u8]> + 'a, + { + HexSlice(data.as_ref()) + } +} + +// You can choose to implement multiple traits, like Lower and UpperHex +impl Debug for HexSlice<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + write!(f, "0x")?; + for byte in self.0 { + write!(f, "{:x}", byte)?; + } + Ok(()) + } +} +/// Extension trait for hex display. +pub trait HexDisplayExt { + fn hex_display(&self) -> HexSlice<'_>; +} + +impl> HexDisplayExt for T { + fn hex_display(&self) -> HexSlice<'_> { + HexSlice::new(self) + } +} + +/// The execution mode. +#[derive(Clone)] +pub enum Mode { + /// Online. + Online(OnlineConfig), + /// Offline. Uses a cached file and needs not any client config. + Offline(OfflineConfig), +} + +/// configuration of the online execution. +/// +/// A cache config must be present. +#[derive(Clone)] +pub struct OfflineConfig { + cache: CacheConfig, +} + +/// Configuration of the online execution. +/// +/// A cache config may be present and will be written to in that case. +#[derive(Clone)] +pub struct OnlineConfig { + uri: String, + at: Option, + cache: Option, + modules: Vec, +} + +impl Default for OnlineConfig { + fn default() -> Self { + Self { + uri: "http://localhost:9933".into(), + at: None, + cache: None, + modules: Default::default(), + } + } +} + +/// Configuration of the cache. +#[derive(Clone)] +pub struct CacheConfig { + name: String, + directory: String, +} + +impl Default for CacheConfig { + fn default() -> Self { + Self { name: "CACHE".into(), directory: ".".into() } + } +} + +impl CacheConfig { + fn path(&self) -> PathBuf { + Path::new(&self.directory).join(self.name.clone()) + } +} + +/// Builder for remote-externalities. +pub struct Builder { + inject: Vec, + mode: Mode, + chain: String, +} + +impl Default for Builder { + fn default() -> Self { + Self { + inject: Default::default(), + mode: Mode::Online(OnlineConfig { + at: None, + uri: "http://localhost:9933".into(), + cache: None, + modules: Default::default(), + }), + chain: "UNSET".into(), + } + } +} + +// Mode methods +impl Builder { + fn as_online(&self) -> &OnlineConfig { + match &self.mode { + Mode::Online(config) => &config, + _ => panic!("Unexpected mode: Online"), + } + } + + fn as_online_mut(&mut self) -> &mut OnlineConfig { + match &mut self.mode { + Mode::Online(config) => config, + _ => panic!("Unexpected mode: Online"), + } + } +} + +// RPC methods +impl Builder { + async fn rpc_get_head(&self) -> Hash { + let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + let uri = self.as_online().uri.clone(); + rt.block_on::<_, _, ()>(futures::lazy(move || { + trace!(target: LOG_TARGET, "rpc: finalized_head"); + let client: sc_rpc_api::chain::ChainClient = + jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); + Ok(client.finalized_head().wait().unwrap()) + })) + .unwrap() + } + + /// Relay the request to `state_getPairs` rpc endpoint. + /// + /// Note that this is an unsafe RPC. + async fn rpc_get_pairs(&self, prefix: StorageKey, at: Hash) -> Vec { + let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + let uri = self.as_online().uri.clone(); + rt.block_on::<_, _, ()>(futures::lazy(move || { + trace!(target: LOG_TARGET, "rpc: storage_pairs: {:?} / {:?}", prefix, at); + let client: sc_rpc_api::state::StateClient = + jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); + Ok(client.storage_pairs(prefix, Some(at)).wait().unwrap()) + })) + .unwrap() + } + + /// Get the chain name. + async fn chain_name(&self) -> String { + let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + let uri = self.as_online().uri.clone(); + rt.block_on::<_, _, ()>(futures::lazy(move || { + trace!(target: LOG_TARGET, "rpc: system_chain"); + let client: sc_rpc_api::system::SystemClient<(), ()> = + jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); + Ok(client.system_chain().wait().unwrap()) + })) + .unwrap() + } +} + +// Internal methods +impl Builder { + /// Save the given data as cache. + fn save_cache(&self, data: &[KeyPair], path: &Path) { + let bdata = bincode::serialize(data).unwrap(); + info!(target: LOG_TARGET, "writing to cache file {:?}", path); + fs::write(path, bdata).unwrap(); + } + + /// initialize `Self` from cache. Panics if the file does not exist. + fn load_cache(&self, path: &Path) -> Vec { + info!(target: LOG_TARGET, "scraping keypairs from cache {:?}", path,); + let bytes = fs::read(path).unwrap(); + bincode::deserialize(&bytes[..]).unwrap() + } + + /// Build `Self` from a network node denoted by `uri`. + async fn load_remote(&self) -> Vec { + let config = self.as_online(); + let at = self.as_online().at.unwrap().clone(); + info!(target: LOG_TARGET, "scraping keypairs from remote node {} @ {:?}", config.uri, at); + + let keys_and_values = if config.modules.len() > 0 { + let mut filtered_kv = vec![]; + for f in config.modules.iter() { + let hashed_prefix = StorageKey(twox_128(f.as_bytes()).to_vec()); + let module_kv = self.rpc_get_pairs(hashed_prefix.clone(), at).await; + info!( + target: LOG_TARGET, + "downloaded data for module {} (count: {} / prefix: {:?}).", + f, + module_kv.len(), + hashed_prefix.hex_display(), + ); + filtered_kv.extend(module_kv); + } + filtered_kv + } else { + info!(target: LOG_TARGET, "downloading data for all modules."); + self.rpc_get_pairs(StorageKey(vec![]), at).await.into_iter().collect::>() + }; + + keys_and_values + } + + async fn init_remote_client(&mut self) { + self.as_online_mut().at = Some(self.rpc_get_head().await); + // TODO: set the at. + self.chain = self.chain_name().await; + } + + async fn pre_build(mut self) -> Vec { + let mut base_kv = match self.mode.clone() { + Mode::Offline(config) => self.load_cache(&config.cache.path()), + Mode::Online(config) => { + self.init_remote_client().await; + let kp = self.load_remote().await; + if let Some(c) = config.cache { + self.save_cache(&kp, &c.path()); + } + kp + } + }; + + base_kv.extend(self.inject.clone()); + base_kv + } +} + +// Public methods +impl Builder { + /// Create a new builder. + pub fn new() -> Self { + Default::default() + } + + /// Inject a manual list of key and values to the storage. + pub fn inject(mut self, injections: &[KeyPair]) -> Self { + for i in injections { + self.inject.push(i.clone()); + } + self + } + + /// Configure a cache to be used. + pub fn mode(mut self, mode: Mode) -> Self { + self.mode = mode; + self + } + + /// Build the test externalities. + pub async fn build(self) -> TestExternalities { + let kv = self.pre_build().await; + let mut ext = TestExternalities::new_empty(); + + info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len()); + for (k, v) in kv { + let (k, v) = (k.0, v.0); + trace!(target: LOG_TARGET, "injecting {:?} -> {:?}", k.hex_display(), v.hex_display()); + ext.insert(k, v); + } + ext + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[async_std::test] + #[cfg(feature = "remote-test")] + async fn can_build_one_pallet() { + let _ = env_logger::Builder::from_default_env() + .format_module_path(false) + .format_level(true) + .try_init(); + + Builder::new() + .mode(Mode::Online(OnlineConfig { + modules: vec!["Proxy".into()], + ..Default::default() + })) + .build() + .await + .execute_with(|| {}); + } + + #[async_std::test] + async fn can_load_cache() { + let _ = env_logger::Builder::from_default_env() + .format_module_path(false) + .format_level(true) + .try_init(); + + Builder::new() + .mode(Mode::Offline(OfflineConfig { + cache: CacheConfig { name: "proxy_test".into(), ..Default::default() }, + })) + .build() + .await + .execute_with(|| {}); + } + + #[async_std::test] + #[cfg(feature = "remote-test")] + async fn can_create_cache() { + let _ = env_logger::Builder::from_default_env() + .format_module_path(false) + .format_level(true) + .try_init(); + + Builder::new() + .mode(Mode::Online(OnlineConfig { + cache: Some(CacheConfig { + name: "test_cache_to_remove.bin".into(), + ..Default::default() + }), + ..Default::default() + })) + .build() + .await + .execute_with(|| {}); + + let to_delete = std::fs::read_dir(CacheConfig::default().directory) + .unwrap() + .into_iter() + .map(|d| d.unwrap()) + .filter(|p| p.path().extension().unwrap_or_default() == "bin") + .collect::>(); + + assert!(to_delete.len() > 0); + + for d in to_delete { + std::fs::remove_file(d.path()).unwrap(); + } + } + + #[async_std::test] + #[cfg(feature = "remote-test")] + async fn can_build_all() { + let _ = env_logger::Builder::from_default_env() + .format_module_path(true) + .format_level(true) + .try_init(); + + Builder::new().build().await.execute_with(|| {}); + } +} From 7f9b5b89a0fa65f061973578aafb93e0b6504df1 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 10 Feb 2021 09:39:12 +0000 Subject: [PATCH 02/18] Checkpoint to move remote. --- Cargo.lock | 27 ++++---- Cargo.toml | 2 +- bin/node/cli/Cargo.toml | 1 + bin/node/cli/src/cli.rs | 6 +- bin/node/cli/src/command.rs | 2 +- bin/node/runtime/Cargo.toml | 6 +- bin/node/runtime/src/lib.rs | 12 ++-- frame/dry-run-runtime-upgrade/Cargo.toml | 29 +++++++++ frame/dry-run-runtime-upgrade/src/lib.rs | 51 +++++++++++++++ frame/executive/Cargo.toml | 1 + frame/executive/src/lib.rs | 35 +++++------ frame/support/Cargo.toml | 1 + frame/support/src/traits.rs | 16 +++-- .../dry-run-runtime-upgrade/api/Cargo.toml | 14 ----- .../dry-run-runtime-upgrade/api/src/lib.rs | 10 --- .../dry-run-runtime-upgrade/cli/Cargo.toml | 14 ++++- .../dry-run-runtime-upgrade/cli/src/lib.rs | 62 ++++++++----------- .../remote-externalities/.gitignore | 1 + .../remote-externalities/Cargo.toml | 12 +++- .../remote-externalities/src/lib.rs | 1 - 20 files changed, 189 insertions(+), 114 deletions(-) create mode 100644 frame/dry-run-runtime-upgrade/Cargo.toml create mode 100644 frame/dry-run-runtime-upgrade/src/lib.rs delete mode 100644 utils/frame/dry-run-runtime-upgrade/api/Cargo.toml delete mode 100644 utils/frame/dry-run-runtime-upgrade/api/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 50a7c41ae3c64..5a83bd8ae427c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1353,18 +1353,13 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" -[[package]] -name = "dry-run-runtime-upgrade-api" -version = "0.1.0" -dependencies = [ - "sp-api", -] - [[package]] name = "dry-run-runtime-upgrade-cli" -version = "2.0.0" +version = "2.0.1" dependencies = [ - "dry-run-runtime-upgrade-api", + "frame-dry-run-runtime-upgrade", + "log", + "parity-scale-codec", "remote-externalities", "sc-cli", "sc-client-api", @@ -1717,6 +1712,16 @@ dependencies = [ "structopt", ] +[[package]] +name = "frame-dry-run-runtime-upgrade" +version = "2.0.1" +dependencies = [ + "frame-support", + "parity-scale-codec", + "sp-api", + "sp-std", +] + [[package]] name = "frame-executive" version = "2.0.1" @@ -4095,8 +4100,8 @@ dependencies = [ name = "node-runtime" version = "2.0.1" dependencies = [ - "dry-run-runtime-upgrade-api", "frame-benchmarking", + "frame-dry-run-runtime-upgrade", "frame-executive", "frame-support", "frame-system", @@ -6387,7 +6392,7 @@ dependencies = [ [[package]] name = "remote-externalities" -version = "0.1.0" +version = "2.0.1" dependencies = [ "async-std", "bincode", diff --git a/Cargo.toml b/Cargo.toml index 14029293ab350..8a7df9f122061 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ members = [ "frame/contracts/rpc", "frame/contracts/rpc/runtime-api", "frame/democracy", + "frame/dry-run-runtime-upgrade", "frame/elections", "frame/example", "frame/example-offchain-worker", @@ -188,7 +189,6 @@ members = [ "utils/frame/frame-utilities-cli", "utils/frame/dry-run-runtime-upgrade/remote-externalities", "utils/frame/dry-run-runtime-upgrade/cli", - "utils/frame/dry-run-runtime-upgrade/api", "utils/frame/rpc/support", "utils/frame/rpc/system", "utils/prometheus", diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 79da5e61ed4b7..d400d9235d66d 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -163,3 +163,4 @@ runtime-benchmarks = [ "node-runtime/runtime-benchmarks", "frame-benchmarking-cli", ] +runtime-upgrade-dry-run = [] diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 6fcc475431076..1c1d7a0a64c83 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -47,8 +47,10 @@ pub enum Subcommand { #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), - /// DryRun all of the runtime upgrade hooks in the current runtime upon a configurable state. - DryRunRuntimeUpgrade(dry_run_runtime_upgrade_cli::DryRunCmd), + /// Dry-run all of the runtime upgrade hooks in the current runtime upon a configurable state. + /// + /// The state is independent of the current chain and can be fetched remotely. + DryRunRuntimeUpgrade(dry_run_runtime_upgrade_cli::DryRunRuntimeUpgradeCmd), /// Verify a signature for a message, provided on STDIN, with a given (public or secret) key. Verify(VerifyCmd), diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 297de09a08a2a..68883816a899a 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -149,7 +149,7 @@ pub fn run() -> Result<()> { Ok((cmd.run(client, backend), task_manager)) }) }, - Some(Subcommand::DryRun(cmd)) => { + Some(Subcommand::DryRunRuntimeUpgrade(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|config| { let PartialComponents { task_manager, .. } = new_partial(&config)?; diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 55f875d0a7773..8360c8cea74fb 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -83,9 +83,7 @@ pallet-transaction-payment = { version = "2.0.0", default-features = false, path pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } pallet-vesting = { version = "2.0.0", default-features = false, path = "../../../frame/vesting" } -dry-run-runtime-upgrade-api = { default-features = false, path = "../../../utils/frame/dry-run-runtime-upgrade/api" } -# TODO: probably best to put this in some primitive pallet, among with types that need to be passed -# back and forth. +frame-dry-run-runtime-upgrade = { default-features = false, path = "../../../frame/dry-run-runtime-upgrade" } [build-dependencies] substrate-wasm-builder = { version = "3.0.0", path = "../../../utils/wasm-builder" } @@ -155,7 +153,7 @@ std = [ "pallet-society/std", "pallet-recovery/std", "pallet-vesting/std", - "dry-run-runtime-upgrade-api/std", + "frame-dry-run-runtime-upgrade/std", ] runtime-benchmarks = [ "frame-benchmarking", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index a2e541ba7d80e..cfa5d709ebe2e 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1330,22 +1330,24 @@ impl_runtime_apis! { } } - // TODO: make everything feature gated. - impl dry_run_runtime_upgrade_api::DryRunRuntimeUpgrade for Runtime { - fn dry_run_runtime_upgrade() -> Weight { + #[cfg(feature = "runtime-upgrade-dry-run")] + impl frame_dry_run_runtime_upgrade::DryRunRuntimeUpgrade for Runtime { + fn dry_run_runtime_upgrade(pallet: &str) -> Weight { frame_support::debug::RuntimeLogger::init(); + frame_support::debug::info!("!!! DRYRUN MIGRATION UP AHEAD !!!"); let weight = Executive::dry_run_runtime_upgrade(); frame_support::debug::info!("!!! DRYRUN MIGRATION DONE !!!"); + weight // \migration_bot pallet:staking state:polkadot // pseudo code: // match config.pallet { // "System" => { - // ::pre_migration(); + // ::pre_upgrade(); // ::on_runtime_upgrade(); - // ::post_migration(); + // ::post_upgrade(); // }, // "All" => { // Executive::dry_run_runtime_upgrade() diff --git a/frame/dry-run-runtime-upgrade/Cargo.toml b/frame/dry-run-runtime-upgrade/Cargo.toml new file mode 100644 index 0000000000000..58827d385cdc7 --- /dev/null +++ b/frame/dry-run-runtime-upgrade/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "frame-dry-run-runtime-upgrade" +version = "2.0.1" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for democracy" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } + +sp-api = { version = "2.0.1", path = "../../primitives/api", default-features = false } +sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } + +frame-support = { version = "2.0.1", path = "../support", default-features = false } + +[features] +default = [ "std" ] +std = [ + "sp-api/std", + "sp-std/std", + "frame-support/std", +] diff --git a/frame/dry-run-runtime-upgrade/src/lib.rs b/frame/dry-run-runtime-upgrade/src/lib.rs new file mode 100644 index 0000000000000..f61f98e00af28 --- /dev/null +++ b/frame/dry-run-runtime-upgrade/src/lib.rs @@ -0,0 +1,51 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Supporting types for runtime upgrade dry-run api and command. + +#![cfg_attr(not(feature = "std"), no_std)] + +use codec::Encode; +use sp_std::prelude::*; + +/// Possible targets for dry-run runtime upgrade. +#[derive(Debug, Encode)] +pub enum Target { + /// All pallets. + All, + /// A single pallet. Inner value is the encoded pallet name. + Pallet(Vec), +} + +#[cfg(feature = "std")] +impl sp_std::str::FromStr for Target { + type Err = &'static str; + fn from_str(s: &str) -> Result { + Ok(if s.to_lowercase() == "All" { Target::All } else { Target::Pallet(s.encode()) }) + } +} + +sp_api::decl_runtime_apis! { + /// Runtime api for testing the execution of an upcoming runtime upgrade. + pub trait DryRunRuntimeUpgrade { + /// dry-run runtime upgrades, returning the total weight consumed. + /// + /// Returns the consumed weight of the migration in case of a successful one, and panics + /// otherwise. + fn dry_run_runtime_upgrade(target: Target) -> frame_support::weights::Weight; + } +} diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml index 6ee378b222ca3..212a1d2107cb0 100644 --- a/frame/executive/Cargo.toml +++ b/frame/executive/Cargo.toml @@ -47,3 +47,4 @@ std = [ "sp-tracing/std", "sp-std/std", ] +runtime-upgrade-dry-run = [] diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index b7a0f91282ce4..07a2f499a2b5e 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -186,25 +186,23 @@ where } impl< - System: frame_system::Config, - Block: traits::Block, - Context: Default, - UnsignedValidator, - AllModules: - OnRuntimeUpgrade + - OnInitialize + - OnFinalize + - OffchainWorker, - COnRuntimeUpgrade: OnRuntimeUpgrade, -> Executive + System: frame_system::Config, + Block: traits::Block
, + Context: Default, + UnsignedValidator, + AllModules: OnRuntimeUpgrade + + OnInitialize + + OnFinalize + + OffchainWorker, + COnRuntimeUpgrade: OnRuntimeUpgrade, + > Executive where Block::Extrinsic: Checkable + Codec, - CheckedOf: - Applyable + - GetDispatchInfo, - CallOf: Dispatchable, + CheckedOf: Applyable + GetDispatchInfo, + CallOf: + Dispatchable, OriginOf: From>, - UnsignedValidator: ValidateUnsigned>, + UnsignedValidator: ValidateUnsigned>, { /// Execute all `OnRuntimeUpgrade` of this runtime, and return the aggregate weight. pub fn do_on_runtime_upgrade() -> frame_support::weights::Weight { @@ -221,12 +219,13 @@ where /// Execute all `OnRuntimeUpgrade` of this runtime, including the pre and post migration checks. /// /// This should only be used for testing. + #[cfg(feature = "runtime-upgrade-dry-run")] pub fn dry_run_runtime_upgrade() -> frame_support::weights::Weight { < (frame_system::Module::, COnRuntimeUpgrade, AllModules) as OnRuntimeUpgrade - >::pre_migration().expect("pre-migration hook failed."); + >::pre_upgrade().expect("pre_upgrade hook failed."); let weight = Self::do_on_runtime_upgrade(); @@ -234,7 +233,7 @@ where (frame_system::Module::, COnRuntimeUpgrade, AllModules) as OnRuntimeUpgrade - >::post_migration().expect("post-migration hook failed."); + >::post_upgrade().expect("post_upgrade hook failed."); weight } diff --git a/frame/support/Cargo.toml b/frame/support/Cargo.toml index 294e4c1574a32..526fd65e28a26 100644 --- a/frame/support/Cargo.toml +++ b/frame/support/Cargo.toml @@ -60,3 +60,4 @@ std = [ nightly = [] strict = [] runtime-benchmarks = [] +runtime-upgrade-dry-run = [] diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 0fbb7a80b798c..b8172c85fc6cd 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -1560,7 +1560,8 @@ pub trait OnRuntimeUpgrade { /// /// These hooks are never meant to be executed on-chain, instead only be used by 3rd party tools /// for testing. - fn pre_migration() -> Result<(), &'static str> { + #[cfg(feature = "runtime-upgrade-dry-run")] + fn pre_upgrade() -> Result<(), &'static str> { Ok(()) } @@ -1568,7 +1569,8 @@ pub trait OnRuntimeUpgrade { /// /// These hooks are never meant to be executed on-chain, instead only be used by 3rd party tools /// for testing. - fn post_migration() -> Result<(), &'static str> { + #[cfg(feature = "runtime-upgrade-dry-run")] + fn post_upgrade() -> Result<(), &'static str> { Ok(()) } } @@ -1581,15 +1583,17 @@ impl OnRuntimeUpgrade for Tuple { weight } - fn pre_migration() -> Result<(), &'static str> { + #[cfg(feature = "runtime-upgrade-dry-run")] + fn pre_upgrade() -> Result<(), &'static str> { let mut result = Ok(()); - for_tuples!( #( result = result.and(Tuple::pre_migration()); )* ); + for_tuples!( #( result = result.and(Tuple::pre_upgrade()); )* ); result } - fn post_migration() -> Result<(), &'static str> { + #[cfg(feature = "runtime-upgrade-dry-run")] + fn post_upgrade() -> Result<(), &'static str> { let mut result = Ok(()); - for_tuples!( #( result = result.and(Tuple::post_migration()); )* ); + for_tuples!( #( result = result.and(Tuple::post_upgrade()); )* ); result } } diff --git a/utils/frame/dry-run-runtime-upgrade/api/Cargo.toml b/utils/frame/dry-run-runtime-upgrade/api/Cargo.toml deleted file mode 100644 index b4eafa6c40e19..0000000000000 --- a/utils/frame/dry-run-runtime-upgrade/api/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "dry-run-runtime-upgrade-api" -version = "0.1.0" -authors = ["kianenigma "] -edition = "2018" - -[dependencies] -sp-api = { version = "2.0.1", path = "../../../../primitives/api", default-features = false } - -[features] -default = [ "std" ] -std = [ - "sp-api/std", -] diff --git a/utils/frame/dry-run-runtime-upgrade/api/src/lib.rs b/utils/frame/dry-run-runtime-upgrade/api/src/lib.rs deleted file mode 100644 index cd53ca97f175a..0000000000000 --- a/utils/frame/dry-run-runtime-upgrade/api/src/lib.rs +++ /dev/null @@ -1,10 +0,0 @@ -#![cfg_attr(not(feature = "std"), no_std)] - -sp_api::decl_runtime_apis! { - /// Runtime api for testing the migration of a FRAME runtime. - // TODO: move this to frame. - pub trait DryRunRuntimeUpgrade { - /// dry-run runtime upgrades, returning the total weight consumed. - fn dry_run_runtime_upgrade() -> u64; - } -} diff --git a/utils/frame/dry-run-runtime-upgrade/cli/Cargo.toml b/utils/frame/dry-run-runtime-upgrade/cli/Cargo.toml index 48e7c9d70066a..9bae6abcf7e1f 100644 --- a/utils/frame/dry-run-runtime-upgrade/cli/Cargo.toml +++ b/utils/frame/dry-run-runtime-upgrade/cli/Cargo.toml @@ -1,13 +1,21 @@ [package] name = "dry-run-runtime-upgrade-cli" -version = "2.0.0" -authors = ["kianenigma "] +version = "2.0.1" +authors = ["Parity Technologies "] edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Cli command for dry-running a runtime upgrade" +readme = "README.md" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] +log = "0.4.8" +parity-scale-codec = { version = "2.0.0" } + sc-service = { version = "0.8.0", default-features = false, path = "../../../../client/service" } sc-cli = { version = "0.8.0", path = "../../../../client/cli" } sc-executor = { path = "../../../../client/executor" } @@ -19,6 +27,6 @@ sp-blockchain = { version = "2.0.1", path = "../../../../primitives/blockchain" sp-runtime = { version = "2.0.1", path = "../../../../primitives/runtime" } sp-externalities = { path = "../../../../primitives/externalities" } sp-core = { path = "../../../../primitives/core" } +frame-dry-run-runtime-upgrade = { path = "../../../../frame/dry-run-runtime-upgrade" } -dry-run-runtime-upgrade-api = { path = "../api" } remote-externalities = { path = "../remote-externalities" } diff --git a/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs b/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs index 02ebe33bb9767..679fbee1f600e 100644 --- a/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs +++ b/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs @@ -15,6 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use parity_scale_codec::{Decode, Encode}; use std::{fmt::Debug, str::FromStr}; use sc_service::Configuration; use sc_cli::{CliConfiguration, ExecutionStrategy}; @@ -23,9 +24,10 @@ use sp_state_machine::StateMachine; use sc_service::NativeExecutionDispatch; use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_core::storage::{StorageData, StorageKey, well_known_keys}; +use frame_dry_run_runtime_upgrade::Target; #[derive(Debug, structopt::StructOpt)] -pub struct DryRunCmd { +pub struct DryRunRuntimeUpgradeCmd { /// The shared parameters #[allow(missing_docs)] #[structopt(flatten)] @@ -35,30 +37,12 @@ pub struct DryRunCmd { #[structopt(short, long, default_value = "All")] pub target: Target, - /// The state to use to run the migration. It should be a ":" separated string, where the - /// prefix is either `live` or `snap`, and the postfix is either the HTTP uri or the file path, - /// respectively. - #[structopt(short, long, default_value = "live:http://localhost:9933")] + /// The state to use to run the migration. Should be a valid FILE or HTTP URI. + #[structopt(short, long, default_value = "http://localhost:9933")] pub state: State, } -/// Possible targets for dryrun runtime upgrade. -#[derive(Debug)] -pub enum Target { - /// All pallets. - All, - /// A single pallet. - Pallet(String), -} - -impl FromStr for Target { - type Err = &'static str; - fn from_str(s: &str) -> Result { - Ok(if s.to_lowercase() == "All" { Target::All } else { Target::Pallet(s.to_string()) }) - } -} - -/// The state to use for a migration dryrun. +/// The state to use for a migration dry-run. #[derive(Debug)] pub enum State { /// A snapshot. Inner value is file path. @@ -71,21 +55,21 @@ pub enum State { impl FromStr for State { type Err = &'static str; fn from_str(s: &str) -> Result { - let splitted = s.splitn(2, ":").collect::>(); - if splitted.len() != 2 { - return Err("invalid format. Must be [state_type]:[state_value]."); - } - let state_type = splitted[0]; - let value = splitted[1]; - match state_type { - "live" => Ok(State::Live(value.to_string())), - "snap" => Ok(State::Snap(value.to_string())), - _ => Err("Invalid state type."), + match s.get(..7) { + // could use Url crate as well, but lets keep it simple for now. + Some("http://") => Ok(State::Live(s.to_string())), + Some("file://") => s + .split("//") + .collect::>() + .get(1) + .map(|s| State::Snap(s.to_string())) + .ok_or("invalid file URI"), + _ => Err("invalid format. Must be a valid HTTP or File URI"), } } } -impl DryRunCmd { +impl DryRunRuntimeUpgradeCmd { pub async fn run(&self, config: Configuration) -> sc_cli::Result<()> where B: BlockT, @@ -106,13 +90,13 @@ impl DryRunCmd { let ext = remote_externalities::Builder::new().inject(&[(code_key, code)]).build().await; - let raw_result = StateMachine::<_, _, NumberFor, _>::new( + let consumed_weight = StateMachine::<_, _, NumberFor, _>::new( &ext.backend, None, &mut changes, &executor, "DryRunRuntimeUpgrade_dry_run_runtime_upgrade", - &[], + &self.target.encode(), ext.extensions, &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) .runtime_code() @@ -122,11 +106,17 @@ impl DryRunCmd { .execute(strategy.into()) .unwrap(); + let weight = ::decode(&mut &*consumed_weight).unwrap(); + log::info!( + "dry-run-runtime-upgraded executed without errors. Consumed weight = {}", + weight, + ); + Ok(()) } } -impl CliConfiguration for DryRunCmd { +impl CliConfiguration for DryRunRuntimeUpgradeCmd { fn shared_params(&self) -> &sc_cli::SharedParams { &self.shared_params } diff --git a/utils/frame/dry-run-runtime-upgrade/remote-externalities/.gitignore b/utils/frame/dry-run-runtime-upgrade/remote-externalities/.gitignore index a8a0dcec44720..75baa697047b8 100644 --- a/utils/frame/dry-run-runtime-upgrade/remote-externalities/.gitignore +++ b/utils/frame/dry-run-runtime-upgrade/remote-externalities/.gitignore @@ -1 +1,2 @@ *.bin +.gitignore diff --git a/utils/frame/dry-run-runtime-upgrade/remote-externalities/Cargo.toml b/utils/frame/dry-run-runtime-upgrade/remote-externalities/Cargo.toml index 7128bfd3eeb35..54ac3b62b7858 100644 --- a/utils/frame/dry-run-runtime-upgrade/remote-externalities/Cargo.toml +++ b/utils/frame/dry-run-runtime-upgrade/remote-externalities/Cargo.toml @@ -1,8 +1,16 @@ [package] name = "remote-externalities" -version = "0.1.0" -authors = ["kianenigma "] +version = "2.0.1" +authors = ["Parity Technologies "] edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "An externalities provided environemnt that can load itself from remote nodes or cache files" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] [dependencies] jsonrpc-core-client = { version = "15.1.0", features = ["http"] } diff --git a/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs b/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs index cefaddb2378e6..26426b4ab61ec 100644 --- a/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs +++ b/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs @@ -339,7 +339,6 @@ impl Builder { async fn init_remote_client(&mut self) { self.as_online_mut().at = Some(self.rpc_get_head().await); - // TODO: set the at. self.chain = self.chain_name().await; } From 488381208643692077b42badf04f0907427f6c29 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 10 Feb 2021 13:08:54 +0000 Subject: [PATCH 03/18] A lot of dependency wiring to make it feature gated. --- bin/node/cli/Cargo.toml | 5 +++- bin/node/runtime/Cargo.toml | 7 +++-- bin/node/runtime/src/lib.rs | 30 ++++++++----------- frame/dry-run-runtime-upgrade/src/lib.rs | 10 +++++-- frame/executive/Cargo.toml | 4 ++- .../dry-run-runtime-upgrade/cli/src/lib.rs | 16 +++++++++- .../remote-externalities/src/lib.rs | 29 +++++++++++++----- 7 files changed, 68 insertions(+), 33 deletions(-) diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index d400d9235d66d..256efcd8dfaf2 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -163,4 +163,7 @@ runtime-benchmarks = [ "node-runtime/runtime-benchmarks", "frame-benchmarking-cli", ] -runtime-upgrade-dry-run = [] +runtime-upgrade-dry-run = [ + "node-runtime/runtime-upgrade-dry-run", + "dry-run-runtime-upgrade-cli", +] diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 8360c8cea74fb..2817f6a3599f1 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -43,6 +43,7 @@ frame-support = { version = "2.0.0", default-features = false, path = "../../../ frame-system = { version = "2.0.0", default-features = false, path = "../../../frame/system" } frame-system-benchmarking = { version = "2.0.0", default-features = false, path = "../../../frame/system/benchmarking", optional = true } frame-system-rpc-runtime-api = { version = "2.0.0", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } +frame-dry-run-runtime-upgrade = { default-features = false, path = "../../../frame/dry-run-runtime-upgrade", optional = true } pallet-assets = { version = "2.0.0", default-features = false, path = "../../../frame/assets" } pallet-authority-discovery = { version = "2.0.0", default-features = false, path = "../../../frame/authority-discovery" } pallet-authorship = { version = "2.0.0", default-features = false, path = "../../../frame/authorship" } @@ -83,8 +84,6 @@ pallet-transaction-payment = { version = "2.0.0", default-features = false, path pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } pallet-vesting = { version = "2.0.0", default-features = false, path = "../../../frame/vesting" } -frame-dry-run-runtime-upgrade = { default-features = false, path = "../../../frame/dry-run-runtime-upgrade" } - [build-dependencies] substrate-wasm-builder = { version = "3.0.0", path = "../../../utils/wasm-builder" } @@ -189,3 +188,7 @@ runtime-benchmarks = [ "frame-system-benchmarking", "hex-literal", ] +runtime-upgrade-dry-run = [ + "frame-executive/runtime-upgrade-dry-run", + "frame-dry-run-runtime-upgrade", +] diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index cfa5d709ebe2e..dcb7396ac2bdd 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1332,27 +1332,23 @@ impl_runtime_apis! { #[cfg(feature = "runtime-upgrade-dry-run")] impl frame_dry_run_runtime_upgrade::DryRunRuntimeUpgrade for Runtime { - fn dry_run_runtime_upgrade(pallet: &str) -> Weight { + fn dry_run_runtime_upgrade(target: frame_dry_run_runtime_upgrade::Target) -> Weight { frame_support::debug::RuntimeLogger::init(); - frame_support::debug::info!("!!! DRYRUN MIGRATION UP AHEAD !!!"); - let weight = Executive::dry_run_runtime_upgrade(); - frame_support::debug::info!("!!! DRYRUN MIGRATION DONE !!!"); - weight + let weight = match target { + frame_dry_run_runtime_upgrade::Target::All => { + frame_support::debug::info!("Dru-running all on-runtime-upgrades."); + Executive::dry_run_runtime_upgrade() + }, + frame_dry_run_runtime_upgrade::Target::Pallet(name) => { + let name = sp_std::str::from_utf8(&name).unwrap(); + frame_support::debug::info!("Dru-running on-runtime-upgrade of {}", name); + todo!(); + } + }; - // \migration_bot pallet:staking state:polkadot - // pseudo code: - // match config.pallet { - // "System" => { - // ::pre_upgrade(); - // ::on_runtime_upgrade(); - // ::post_upgrade(); - // }, - // "All" => { - // Executive::dry_run_runtime_upgrade() - // } - // } + weight } } diff --git a/frame/dry-run-runtime-upgrade/src/lib.rs b/frame/dry-run-runtime-upgrade/src/lib.rs index f61f98e00af28..873f847ae5e1c 100644 --- a/frame/dry-run-runtime-upgrade/src/lib.rs +++ b/frame/dry-run-runtime-upgrade/src/lib.rs @@ -19,11 +19,11 @@ #![cfg_attr(not(feature = "std"), no_std)] -use codec::Encode; +use codec::{Decode, Encode}; use sp_std::prelude::*; /// Possible targets for dry-run runtime upgrade. -#[derive(Debug, Encode)] +#[derive(Debug, Encode, Decode)] pub enum Target { /// All pallets. All, @@ -35,7 +35,11 @@ pub enum Target { impl sp_std::str::FromStr for Target { type Err = &'static str; fn from_str(s: &str) -> Result { - Ok(if s.to_lowercase() == "All" { Target::All } else { Target::Pallet(s.encode()) }) + Ok(if s.to_lowercase() == "All" { + Target::All + } else { + Target::Pallet(s.as_bytes().to_vec()) + }) } } diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml index 212a1d2107cb0..8553933dc036c 100644 --- a/frame/executive/Cargo.toml +++ b/frame/executive/Cargo.toml @@ -47,4 +47,6 @@ std = [ "sp-tracing/std", "sp-std/std", ] -runtime-upgrade-dry-run = [] +runtime-upgrade-dry-run = [ + "frame-support/runtime-upgrade-dry-run" +] diff --git a/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs b/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs index 679fbee1f600e..df54fc78bd722 100644 --- a/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs +++ b/utils/frame/dry-run-runtime-upgrade/cli/src/lib.rs @@ -88,7 +88,20 @@ impl DryRunRuntimeUpgradeCmd { let heap_pages = Some(1024); let executor = NativeExecutor::::new(wasm_method, heap_pages, 2); - let ext = remote_externalities::Builder::new().inject(&[(code_key, code)]).build().await; + let ext = { + use remote_externalities::{Builder, Mode, CacheConfig, OfflineConfig, OnlineConfig}; + let builder = match &self.state { + State::Snap(file_path) => Builder::new().mode(Mode::Offline(OfflineConfig { + cache: CacheConfig { name: file_path.into(), ..Default::default() }, + })), + State::Live(http_uri) => Builder::new().mode(Mode::Online(OnlineConfig { + uri: http_uri.into(), + ..Default::default() + })), + }; + + builder.inject(&[(code_key, code)]).build().await + }; let consumed_weight = StateMachine::<_, _, NumberFor, _>::new( &ext.backend, @@ -105,6 +118,7 @@ impl DryRunRuntimeUpgradeCmd { ) .execute(strategy.into()) .unwrap(); + // TODO: if MethodNotFound, then it must be wrong feature. let weight = ::decode(&mut &*consumed_weight).unwrap(); log::info!( diff --git a/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs b/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs index 26426b4ab61ec..d774764e5db85 100644 --- a/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs +++ b/utils/frame/dry-run-runtime-upgrade/remote-externalities/src/lib.rs @@ -166,7 +166,8 @@ pub enum Mode { /// A cache config must be present. #[derive(Clone)] pub struct OfflineConfig { - cache: CacheConfig, + /// The configuration of the cache file to use. It must be present. + pub cache: CacheConfig, } /// Configuration of the online execution. @@ -174,10 +175,14 @@ pub struct OfflineConfig { /// A cache config may be present and will be written to in that case. #[derive(Clone)] pub struct OnlineConfig { - uri: String, - at: Option, - cache: Option, - modules: Vec, + /// The HTTP uri to use. + pub uri: String, + /// The block number at which to connect. Will be latest finalized head if not provided. + pub at: Option, + /// An optional cache file to WRITE to, not for reading. Not cached if set to `None`. + pub cache: Option, + /// The modules to scrape. If empty, entire chain state will be scraped. + pub modules: Vec, } impl Default for OnlineConfig { @@ -194,8 +199,12 @@ impl Default for OnlineConfig { /// Configuration of the cache. #[derive(Clone)] pub struct CacheConfig { - name: String, - directory: String, + // TODO: I could mix these two into one filed, but I think separate is better bc one can be + // configurable while one not. + /// File name. + pub name: String, + /// Base directory. + pub directory: String, } impl Default for CacheConfig { @@ -355,6 +364,11 @@ impl Builder { } }; + info!( + target: LOG_TARGET, + "extending externalities with {} manually injected keys", + self.inject.len() + ); base_kv.extend(self.inject.clone()); base_kv } @@ -389,7 +403,6 @@ impl Builder { info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len()); for (k, v) in kv { let (k, v) = (k.0, v.0); - trace!(target: LOG_TARGET, "injecting {:?} -> {:?}", k.hex_display(), v.hex_display()); ext.insert(k, v); } ext From be549b455d01419395f2d11172bd0bbcf84e2a11 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 10 Feb 2021 16:23:18 +0000 Subject: [PATCH 04/18] bad macro, bad macro. --- bin/node/runtime/src/lib.rs | 17 +++++++++++++---- frame/dry-run-runtime-upgrade/src/lib.rs | 24 +++++++++++++++++++++++- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index dcb7396ac2bdd..040d3d05fe2de 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1334,17 +1334,26 @@ impl_runtime_apis! { impl frame_dry_run_runtime_upgrade::DryRunRuntimeUpgrade for Runtime { fn dry_run_runtime_upgrade(target: frame_dry_run_runtime_upgrade::Target) -> Weight { frame_support::debug::RuntimeLogger::init(); - frame_support::debug::info!("!!! DRYRUN MIGRATION UP AHEAD !!!"); let weight = match target { frame_dry_run_runtime_upgrade::Target::All => { - frame_support::debug::info!("Dru-running all on-runtime-upgrades."); + frame_support::debug::info!("Dry-running all on-runtime-upgrades."); Executive::dry_run_runtime_upgrade() }, frame_dry_run_runtime_upgrade::Target::Pallet(name) => { let name = sp_std::str::from_utf8(&name).unwrap(); - frame_support::debug::info!("Dru-running on-runtime-upgrade of {}", name); - todo!(); + frame_support::debug::info!("Dry-running on-runtime-upgrade of {}.", name); + + frame_dry_run_runtime_upgrade::match_pallet_on_runtime_upgrade!(name, + System, Utility, Babe, Timestamp, Authorship, Indices, Balances, + TransactionPayment, Staking, Session, Democracy, Council, + TechnicalCommittee, Elections, TechnicalMembership, Grandpa, Treasury, + Contracts, Sudo, ImOnline, AuthorityDiscovery, Offences, Historical, + RandomnessCollectiveFlip, Identity, Society, Recovery, Vesting, Scheduler, + Proxy, Multisig, Bounties, Tips, Assets, Mmr, Lottery, + ) + // TODO: ^^ this can be done better, or generated by construct runtime at the + // least. } }; diff --git a/frame/dry-run-runtime-upgrade/src/lib.rs b/frame/dry-run-runtime-upgrade/src/lib.rs index 873f847ae5e1c..b5ead3214c107 100644 --- a/frame/dry-run-runtime-upgrade/src/lib.rs +++ b/frame/dry-run-runtime-upgrade/src/lib.rs @@ -22,6 +22,28 @@ use codec::{Decode, Encode}; use sp_std::prelude::*; +#[doc(hidden)] +pub use frame_support as _support; + +/// Helper macro to generate the match expression needed to match pallet name to their +/// `on_runtime_upgrade()` implementation. +#[macro_export] +macro_rules! match_pallet_on_runtime_upgrade { + ($name:ident, $($pallet:ty),* $(,)*) => { + match $name { + $( + stringify!($pallet) => { + <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::pre_upgrade().unwrap(); + let weight = <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::on_runtime_upgrade(); + <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::post_upgrade().unwrap(); + weight + }, + )* + _ => panic!("Unknown pallet name provided"), + } + }; +} + /// Possible targets for dry-run runtime upgrade. #[derive(Debug, Encode, Decode)] pub enum Target { @@ -35,7 +57,7 @@ pub enum Target { impl sp_std::str::FromStr for Target { type Err = &'static str; fn from_str(s: &str) -> Result { - Ok(if s.to_lowercase() == "All" { + Ok(if s.to_lowercase() == "all" { Target::All } else { Target::Pallet(s.as_bytes().to_vec()) From d84dad48935a674b7660294564f8c579594bcb36 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 11 Feb 2021 14:25:13 +0000 Subject: [PATCH 05/18] Undo the DB mess. --- bin/node/cli/Cargo.toml | 1 - bin/node/cli/src/cli.rs | 5 ++-- bin/node/cli/src/command.rs | 11 +++++++- bin/node/runtime/src/lib.rs | 2 +- client/service/src/task_manager/mod.rs | 2 +- client/state-db/src/lib.rs | 3 ++- frame/executive/src/lib.rs | 2 +- frame/try-runtime/src/lib.rs | 4 +-- utils/frame/try-runtime/cli/Cargo.toml | 2 +- utils/frame/try-runtime/cli/src/lib.rs | 8 +++++- .../remote-externalities/Cargo.toml | 8 +++--- .../remote-externalities/src/lib.rs | 27 +++++++------------ 12 files changed, 40 insertions(+), 35 deletions(-) diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index c4f490917d5c8..76fadb5b44fbf 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -157,7 +157,6 @@ cli = [ "sc-finality-grandpa-warp-sync", "structopt", "substrate-build-script-utils", - "try-runtime-cli", ] runtime-benchmarks = [ "node-runtime/runtime-benchmarks", diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 2aecc192ed389..5c7633e08acd7 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -47,9 +47,8 @@ pub enum Subcommand { #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), - /// Dry-run all of the runtime upgrade hooks in the current runtime upon a configurable state. - /// - /// The state is independent of the current chain and can be fetched remotely. + /// Try some experimental command on the runtime. This includes migration and runtime-upgrade + /// testing. TryRuntime(try_runtime_cli::TryRuntimeCmd), /// Verify a signature for a message, provided on STDIN, with a given (public or secret) key. diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index a5864f032909b..50566bce24790 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -152,7 +152,16 @@ pub fn run() -> Result<()> { Some(Subcommand::TryRuntime(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|config| { - let PartialComponents { task_manager, .. } = new_partial(&config)?; + use sc_service::TaskManager; + // we don't need any of the components of new_partial, just a runtime, or a task + // manager to do `async_run`. + let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry); + let task_manager = TaskManager::new( + config.task_executor.clone(), + registry, + config.telemetry_span.clone(), + ).unwrap(); + Ok((cmd.run::(config), task_manager)) }) } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index c2dc0635f2922..e843ca204476c 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1340,7 +1340,7 @@ impl_runtime_apis! { let weight = match target { frame_try_runtime::Target::All => { frame_support::debug::info!("Dry-running all on-runtime-upgrades."); - Executive::dry_run_runtime_upgrade() + Executive::try_runtime_upgrade() }, frame_try_runtime::Target::Pallet(name) => { let name = sp_std::str::from_utf8(&name).unwrap(); diff --git a/client/service/src/task_manager/mod.rs b/client/service/src/task_manager/mod.rs index 9a1fd15952e12..e7163047f89f4 100644 --- a/client/service/src/task_manager/mod.rs +++ b/client/service/src/task_manager/mod.rs @@ -273,7 +273,7 @@ pub struct TaskManager { impl TaskManager { /// If a Prometheus registry is passed, it will be used to report statistics about the /// service tasks. - pub(super) fn new( + pub fn new( executor: TaskExecutor, prometheus_registry: Option<&Registry>, telemetry_span: Option, diff --git a/client/state-db/src/lib.rs b/client/state-db/src/lib.rs index a8a6bd7bc9437..1f73f3cca35e9 100644 --- a/client/state-db/src/lib.rs +++ b/client/state-db/src/lib.rs @@ -237,7 +237,8 @@ impl StateDbSync Ok(()), - _ => Ok(()), + Some(v) => Err(Error::InvalidPruningMode(String::from_utf8_lossy(v).into())), + None => Ok(()), } } diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index e21b0f9832db4..5af3bed47bd73 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -220,7 +220,7 @@ where /// /// This should only be used for testing. #[cfg(feature = "try-runtime")] - pub fn dry_run_runtime_upgrade() -> frame_support::weights::Weight { + pub fn try_runtime_upgrade() -> frame_support::weights::Weight { < (frame_system::Module::, COnRuntimeUpgrade, AllModules) as diff --git a/frame/try-runtime/src/lib.rs b/frame/try-runtime/src/lib.rs index 0212432a8fc58..d2fd96be15748 100644 --- a/frame/try-runtime/src/lib.rs +++ b/frame/try-runtime/src/lib.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Supporting types for runtime upgrade dry-run api and command. +//! Supporting types for try-runtime, testing and dry-run commands. #![cfg_attr(not(feature = "std"), no_std)] @@ -44,7 +44,7 @@ macro_rules! match_pallet_on_runtime_upgrade { }; } -/// Possible targets for dry-run runtime upgrade. +/// Possible targets for try-runtime testing. #[derive(Debug, Encode, Decode)] pub enum Target { /// All pallets. diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml index 6bc7372dc031c..17d0367d9a703 100644 --- a/utils/frame/try-runtime/cli/Cargo.toml +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" license = "Apache-2.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" -description = "Cli command for dry-running a runtime upgrade" +description = "Cli command runtime testing and dry-running" readme = "README.md" [package.metadata.docs.rs] diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index eb7891cf6906f..3555d52f2a252 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -15,6 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! `Structopt`-ready struct for `try-runtime`. + use parity_scale_codec::{Decode, Encode}; use std::{fmt::Debug, str::FromStr}; use sc_service::Configuration; @@ -79,6 +81,11 @@ impl TryRuntimeCmd { B: BlockT, ExecDispatch: NativeExecutionDispatch + 'static, { + // // prevent a potentially confusing MethodNotFound error from state machine. + // if !cfg!(feature = "try-runtime") { + // panic!("`TryRuntime` api is not enabled without `try-runtime` feature. Compile with this feature.") + // } + let spec = config.chain_spec; let genesis_storage = spec.build_storage()?; @@ -122,7 +129,6 @@ impl TryRuntimeCmd { ) .execute(strategy.into()) .unwrap(); - // TODO: if MethodNotFound, then it must be wrong feature. let weight = ::decode(&mut &*consumed_weight).unwrap(); log::info!( diff --git a/utils/frame/try-runtime/remote-externalities/Cargo.toml b/utils/frame/try-runtime/remote-externalities/Cargo.toml index c7626619a96af..5719aa67ad4b2 100644 --- a/utils/frame/try-runtime/remote-externalities/Cargo.toml +++ b/utils/frame/try-runtime/remote-externalities/Cargo.toml @@ -14,8 +14,8 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] jsonrpc-core-client = { version = "15.1.0", features = ["http"] } -sc-rpc-api = { path = "../../../../client/rpc-api" } -sc-rpc = { path = "../../../../client/rpc" } +sc-rpc-api = { version = "0.9.0", path = "../../../../client/rpc-api" } +sc-rpc = { version = "3.0.0", path = "../../../../client/rpc" } futures = "0.1.29" hex-literal = "0.3.1" @@ -24,8 +24,8 @@ log = "0.4.11" bincode = "1.3.1" tokio = "0.1.22" -sp-io = { path = "../../../../primitives/io" } -sp-core = { path = "../../../../primitives/core" } +sp-io = { version = "3.0.0", path = "../../../../primitives/io" } +sp-core = { version = "3.0.0", path = "../../../../primitives/core" } [dev-dependencies] async-std = { version = "1.6.5", features = ["attributes"] } diff --git a/utils/frame/try-runtime/remote-externalities/src/lib.rs b/utils/frame/try-runtime/remote-externalities/src/lib.rs index d774764e5db85..8d5c19686f8d7 100644 --- a/utils/frame/try-runtime/remote-externalities/src/lib.rs +++ b/utils/frame/try-runtime/remote-externalities/src/lib.rs @@ -413,14 +413,17 @@ impl Builder { mod tests { use super::*; - #[async_std::test] - #[cfg(feature = "remote-test")] - async fn can_build_one_pallet() { + fn init_logger() { let _ = env_logger::Builder::from_default_env() .format_module_path(false) .format_level(true) .try_init(); + } + #[async_std::test] + #[cfg(feature = "remote-test")] + async fn can_build_one_pallet() { + init_logger(); Builder::new() .mode(Mode::Online(OnlineConfig { modules: vec!["Proxy".into()], @@ -433,11 +436,7 @@ mod tests { #[async_std::test] async fn can_load_cache() { - let _ = env_logger::Builder::from_default_env() - .format_module_path(false) - .format_level(true) - .try_init(); - + init_logger(); Builder::new() .mode(Mode::Offline(OfflineConfig { cache: CacheConfig { name: "proxy_test".into(), ..Default::default() }, @@ -450,11 +449,7 @@ mod tests { #[async_std::test] #[cfg(feature = "remote-test")] async fn can_create_cache() { - let _ = env_logger::Builder::from_default_env() - .format_module_path(false) - .format_level(true) - .try_init(); - + init_logger(); Builder::new() .mode(Mode::Online(OnlineConfig { cache: Some(CacheConfig { @@ -484,11 +479,7 @@ mod tests { #[async_std::test] #[cfg(feature = "remote-test")] async fn can_build_all() { - let _ = env_logger::Builder::from_default_env() - .format_module_path(true) - .format_level(true) - .try_init(); - + init_logger(); Builder::new().build().await.execute_with(|| {}); } } From aeb7a0ef4dc3220c5112fa469da1f5fbc57e4825 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Thu, 11 Feb 2021 18:49:56 +0000 Subject: [PATCH 06/18] Update frame/support/src/traits.rs Co-authored-by: Alexander Popiak --- frame/support/src/traits.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 5960084054068..7b9cbb0416daa 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -1560,7 +1560,7 @@ pub trait OnRuntimeUpgrade { Ok(()) } - /// Execute some post-checks prior to a runtime upgrade. + /// Execute some post-checks after a runtime upgrade. /// /// These hooks are never meant to be executed on-chain, instead only be used by 3rd party tools /// for testing. From d968f582ab5f8a7db51a6b08d0c54a63e375199f Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Thu, 11 Feb 2021 18:55:32 +0000 Subject: [PATCH 07/18] Apply suggestions from code review Co-authored-by: Alexander Popiak --- frame/support/src/traits.rs | 6 ++---- frame/try-runtime/Cargo.toml | 2 +- frame/try-runtime/src/lib.rs | 8 ++++---- utils/frame/try-runtime/cli/src/lib.rs | 11 +++-------- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 7b9cbb0416daa..602bc1aa1a690 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -1553,8 +1553,7 @@ pub trait OnRuntimeUpgrade { /// Execute some pre-checks prior to a runtime upgrade. /// - /// These hooks are never meant to be executed on-chain, instead only be used by 3rd party tools - /// for testing. + /// This hook is never meant to be executed on-chain but is meant to be used by testing tools. #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result<(), &'static str> { Ok(()) @@ -1562,8 +1561,7 @@ pub trait OnRuntimeUpgrade { /// Execute some post-checks after a runtime upgrade. /// - /// These hooks are never meant to be executed on-chain, instead only be used by 3rd party tools - /// for testing. + /// This hook is never meant to be executed on-chain but is meant to be used by testing tools. #[cfg(feature = "try-runtime")] fn post_upgrade() -> Result<(), &'static str> { Ok(()) diff --git a/frame/try-runtime/Cargo.toml b/frame/try-runtime/Cargo.toml index b3fa8f3e8c4d5..2bc91328e1136 100644 --- a/frame/try-runtime/Cargo.toml +++ b/frame/try-runtime/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } sp-api = { version = "3.0.0", path = "../../primitives/api", default-features = false } -sp-std = { version = "3.0.0", default-features = false, path = "../../primitives/std" } +sp-std = { version = "3.0.0", path = "../../primitives/std" , default-features = false } frame-support = { version = "3.0.0", path = "../support", default-features = false } diff --git a/frame/try-runtime/src/lib.rs b/frame/try-runtime/src/lib.rs index d2fd96be15748..26d7ae8cda013 100644 --- a/frame/try-runtime/src/lib.rs +++ b/frame/try-runtime/src/lib.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Supporting types for try-runtime, testing and dry-run commands. +//! Supporting types for try-runtime, testing and dry-running commands. #![cfg_attr(not(feature = "std"), no_std)] @@ -25,8 +25,8 @@ use sp_std::prelude::*; #[doc(hidden)] pub use frame_support as _support; -/// Helper macro to generate the match expression needed to match pallet name to their -/// `on_runtime_upgrade()` implementation. +/// Helper macro to generate the match expression needed to match a pallet name to +/// its `on_runtime_upgrade()` implementation. #[macro_export] macro_rules! match_pallet_on_runtime_upgrade { ($name:ident, $($pallet:ty),* $(,)*) => { @@ -66,7 +66,7 @@ impl sp_std::str::FromStr for Target { } sp_api::decl_runtime_apis! { - /// Runtime api for testing the execution of an upcoming runtime upgrade. + /// Runtime api for testing the execution of a runtime upgrade. pub trait TryRuntime { /// dry-run runtime upgrades, returning the total weight consumed. /// diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 3555d52f2a252..61869748c63ca 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -28,10 +28,9 @@ use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_core::storage::{StorageData, StorageKey, well_known_keys}; use frame_try_runtime::Target; -/// Various commands to try-out (similar to `TryFrom`, `TryInto`) the new runtime, over configurable -/// states. +/// Various commands to try out the new runtime, over configurable states. /// -/// For now this only assumes running the runtime-upgrade hooks. +/// For now this only assumes running the `on_runtime_upgrade` hooks. #[derive(Debug, structopt::StructOpt)] pub struct TryRuntimeCmd { /// The shared parameters @@ -51,7 +50,7 @@ pub struct TryRuntimeCmd { /// The state to use for a migration dry-run. #[derive(Debug)] pub enum State { - /// A snapshot. Inner value is file path. + /// A snapshot. Inner value is a file path. Snap(String), /// A live chain. Inner value is the HTTP uri. @@ -81,10 +80,6 @@ impl TryRuntimeCmd { B: BlockT, ExecDispatch: NativeExecutionDispatch + 'static, { - // // prevent a potentially confusing MethodNotFound error from state machine. - // if !cfg!(feature = "try-runtime") { - // panic!("`TryRuntime` api is not enabled without `try-runtime` feature. Compile with this feature.") - // } let spec = config.chain_spec; let genesis_storage = spec.build_storage()?; From ce4128b6bafa68f3668b19380bb53fde02f3df5a Mon Sep 17 00:00:00 2001 From: kianenigma Date: Sat, 13 Feb 2021 19:58:58 +0000 Subject: [PATCH 08/18] unbreak the build --- bin/node/cli/Cargo.toml | 1 + frame/executive/src/lib.rs | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 76fadb5b44fbf..c4f490917d5c8 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -157,6 +157,7 @@ cli = [ "sc-finality-grandpa-warp-sync", "structopt", "substrate-build-script-utils", + "try-runtime-cli", ] runtime-benchmarks = [ "node-runtime/runtime-benchmarks", diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 5af3bed47bd73..690e53f72ebf2 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -205,7 +205,7 @@ where UnsignedValidator: ValidateUnsigned>, { /// Execute all `OnRuntimeUpgrade` of this runtime, and return the aggregate weight. - pub fn do_on_runtime_upgrade() -> frame_support::weights::Weight { + pub fn execute_on_runtime_upgrade() -> frame_support::weights::Weight { let mut weight = 0; weight = weight.saturating_add( as OnRuntimeUpgrade>::on_runtime_upgrade(), @@ -227,7 +227,7 @@ where OnRuntimeUpgrade >::pre_upgrade().expect("pre_upgrade hook failed."); - let weight = Self::do_on_runtime_upgrade(); + let weight = Self::execute_on_runtime_upgrade(); < (frame_system::Module::, COnRuntimeUpgrade, AllModules) @@ -267,7 +267,7 @@ where ) { let mut weight = 0; if Self::runtime_upgraded() { - weight = weight.saturating_add(Self::do_on_runtime_upgrade()); + weight = weight.saturating_add(Self::execute_on_runtime_upgrade()); } >::initialize( block_number, From 712c24091f3ca5bc4cbbb08deb7491af0102f7f1 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Thu, 18 Feb 2021 09:28:36 +0000 Subject: [PATCH 09/18] Update frame/try-runtime/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- frame/try-runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/try-runtime/src/lib.rs b/frame/try-runtime/src/lib.rs index 26d7ae8cda013..163d5bc67f3a6 100644 --- a/frame/try-runtime/src/lib.rs +++ b/frame/try-runtime/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); From 9a23940409bba676d816e7c1b8f97a6f3fd3a2b6 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Thu, 18 Feb 2021 10:11:53 +0000 Subject: [PATCH 10/18] Update utils/frame/try-runtime/cli/Cargo.toml Co-authored-by: Shawn Tabrizi --- utils/frame/try-runtime/cli/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml index 17d0367d9a703..b81b7cd07b0fa 100644 --- a/utils/frame/try-runtime/cli/Cargo.toml +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "try-runtime-cli" -version = "3.0.0" +version = "0.8.0" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" From 93f299aa117766491a5838f3821dce6f3696bb62 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Thu, 18 Feb 2021 10:12:01 +0000 Subject: [PATCH 11/18] Update frame/try-runtime/Cargo.toml Co-authored-by: Shawn Tabrizi --- frame/try-runtime/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/try-runtime/Cargo.toml b/frame/try-runtime/Cargo.toml index 2bc91328e1136..cb7deb156f30d 100644 --- a/frame/try-runtime/Cargo.toml +++ b/frame/try-runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-try-runtime" -version = "3.0.0" +version = "0.8.0" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" From 3cea84066213ac89b18fc39e5b435a13f761c21e Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 18 Feb 2021 10:37:46 +0000 Subject: [PATCH 12/18] Address most review grumbles. --- Cargo.lock | 1 + bin/node/cli/src/cli.rs | 1 + bin/node/cli/src/command.rs | 1 + bin/node/runtime/Cargo.toml | 1 - bin/node/runtime/src/lib.rs | 10 ++- frame/executive/src/lib.rs | 8 +-- frame/try-runtime/Cargo.toml | 4 +- frame/try-runtime/src/lib.rs | 11 +-- primitives/core/src/hexdisplay.rs | 18 +++++ utils/frame/try-runtime/cli/Cargo.toml | 2 +- utils/frame/try-runtime/cli/src/lib.rs | 68 ++++++++++++++----- .../remote-externalities/Cargo.toml | 2 +- .../remote-externalities/src/lib.rs | 39 ++--------- 13 files changed, 95 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 036f112c281f6..67f7e049b95fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1868,6 +1868,7 @@ dependencies = [ "frame-support", "parity-scale-codec", "sp-api", + "sp-runtime", "sp-std", ] diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 5c7633e08acd7..9b80a3e345290 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -49,6 +49,7 @@ pub enum Subcommand { /// Try some experimental command on the runtime. This includes migration and runtime-upgrade /// testing. + #[cfg(feature = "try-runtime")] TryRuntime(try_runtime_cli::TryRuntimeCmd), /// Verify a signature for a message, provided on STDIN, with a given (public or secret) key. diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 50566bce24790..79fb2af860b43 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -149,6 +149,7 @@ pub fn run() -> Result<()> { Ok((cmd.run(client, backend), task_manager)) }) }, + #[cfg(feature = "try-runtime")] Some(Subcommand::TryRuntime(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|config| { diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 759a7b31c7ce7..ebead89ff8b13 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -152,7 +152,6 @@ std = [ "pallet-society/std", "pallet-recovery/std", "pallet-vesting/std", - "frame-try-runtime/std", ] runtime-benchmarks = [ "frame-benchmarking", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 3bd263ff51bba..34616b8d89960 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1334,16 +1334,16 @@ impl_runtime_apis! { #[cfg(feature = "try-runtime")] impl frame_try_runtime::TryRuntime for Runtime { - fn on_runtime_upgrade(target: frame_try_runtime::Target) -> Weight { + fn on_runtime_upgrade(target: frame_try_runtime::Target) -> Result<(Weight, Weight), sp_runtime::RuntimeString> { frame_support::debug::RuntimeLogger::init(); let weight = match target { frame_try_runtime::Target::All => { frame_support::debug::info!("Dry-running all on-runtime-upgrades."); - Executive::try_runtime_upgrade() + Executive::try_runtime_upgrade()? }, frame_try_runtime::Target::Pallet(name) => { - let name = sp_std::str::from_utf8(&name).unwrap(); + let name = sp_std::str::from_utf8(&name).expect("pallet name is utf8-decodable; qed"); frame_support::debug::info!("Dry-running on-runtime-upgrade of {}.", name); frame_try_runtime::match_pallet_on_runtime_upgrade!(name, @@ -1354,12 +1354,10 @@ impl_runtime_apis! { RandomnessCollectiveFlip, Identity, Society, Recovery, Vesting, Scheduler, Proxy, Multisig, Bounties, Tips, Assets, Mmr, Lottery, ) - // TODO: ^^ this can be done better, or generated by construct runtime at the - // least. } }; - weight + Ok((weight, RuntimeBlockWeights::get().max_block)) } } diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 690e53f72ebf2..4e61c36c65a61 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -220,12 +220,12 @@ where /// /// This should only be used for testing. #[cfg(feature = "try-runtime")] - pub fn try_runtime_upgrade() -> frame_support::weights::Weight { + pub fn try_runtime_upgrade() -> Result { < (frame_system::Module::, COnRuntimeUpgrade, AllModules) as OnRuntimeUpgrade - >::pre_upgrade().expect("pre_upgrade hook failed."); + >::pre_upgrade()?; let weight = Self::execute_on_runtime_upgrade(); @@ -233,9 +233,9 @@ where (frame_system::Module::, COnRuntimeUpgrade, AllModules) as OnRuntimeUpgrade - >::post_upgrade().expect("post_upgrade hook failed."); + >::post_upgrade()?; - weight + Ok(weight) } /// Start the execution of a particular block. diff --git a/frame/try-runtime/Cargo.toml b/frame/try-runtime/Cargo.toml index 2bc91328e1136..9c1919d380b82 100644 --- a/frame/try-runtime/Cargo.toml +++ b/frame/try-runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-try-runtime" -version = "3.0.0" +version = "0.9.0" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -17,6 +17,7 @@ codec = { package = "parity-scale-codec", version = "2.0.0", default-features = sp-api = { version = "3.0.0", path = "../../primitives/api", default-features = false } sp-std = { version = "3.0.0", path = "../../primitives/std" , default-features = false } +sp-runtime = { version = "3.0.0", path = "../../primitives/runtime" , default-features = false } frame-support = { version = "3.0.0", path = "../support", default-features = false } @@ -25,5 +26,6 @@ default = [ "std" ] std = [ "sp-api/std", "sp-std/std", + "sp-runtime/std", "frame-support/std", ] diff --git a/frame/try-runtime/src/lib.rs b/frame/try-runtime/src/lib.rs index 26d7ae8cda013..92822a794982c 100644 --- a/frame/try-runtime/src/lib.rs +++ b/frame/try-runtime/src/lib.rs @@ -21,6 +21,7 @@ use codec::{Decode, Encode}; use sp_std::prelude::*; +use frame_support::weights::Weight; #[doc(hidden)] pub use frame_support as _support; @@ -33,9 +34,9 @@ macro_rules! match_pallet_on_runtime_upgrade { match $name { $( stringify!($pallet) => { - <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::pre_upgrade().unwrap(); + <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::pre_upgrade()?; let weight = <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::on_runtime_upgrade(); - <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::post_upgrade().unwrap(); + <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::post_upgrade()?; weight }, )* @@ -70,8 +71,8 @@ sp_api::decl_runtime_apis! { pub trait TryRuntime { /// dry-run runtime upgrades, returning the total weight consumed. /// - /// Returns the consumed weight of the migration in case of a successful one, and panics - /// otherwise. - fn on_runtime_upgrade(target: Target) -> frame_support::weights::Weight; + /// Returns the consumed weight of the migration in case of a successful one, combined with + /// the total allowed block weight of the runtime. + fn on_runtime_upgrade(target: Target) -> Result<(Weight, Weight), sp_runtime::RuntimeString>; } } diff --git a/primitives/core/src/hexdisplay.rs b/primitives/core/src/hexdisplay.rs index 304b665a72c9b..211b9a97917f4 100644 --- a/primitives/core/src/hexdisplay.rs +++ b/primitives/core/src/hexdisplay.rs @@ -53,6 +53,18 @@ impl<'a> sp_std::fmt::Debug for HexDisplay<'a> { } } +/// Extension trait for wrapping a [`HexDisplay`]. +pub trait HexDisplayExt { + /// Display self as hex string. + fn hex_display(&self) -> HexDisplay<'_>; +} + +impl HexDisplayExt for T { + fn hex_display(&self) -> HexDisplay<'_> { + HexDisplay::from(self) + } +} + /// Simple trait to transform various types to `&[u8]` pub trait AsBytesRef { /// Transform `self` into `&[u8]`. @@ -71,6 +83,12 @@ impl AsBytesRef for sp_std::vec::Vec { fn as_bytes_ref(&self) -> &[u8] { &self } } +impl AsBytesRef for sp_storage::StorageKey { + fn as_bytes_ref(&self) -> &[u8] { + self.as_ref() + } +} + macro_rules! impl_non_endians { ( $( $t:ty ),* ) => { $( impl AsBytesRef for $t { diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml index 17d0367d9a703..cdc0a96a3ca58 100644 --- a/utils/frame/try-runtime/cli/Cargo.toml +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -29,4 +29,4 @@ sp-externalities = { version = "0.9.0", path = "../../../../primitives/externali sp-core = { version = "3.0.0", path = "../../../../primitives/core" } frame-try-runtime = { version = "3.0.0", path = "../../../../frame/try-runtime" } -remote-externalities = { verson = "3.0.0", path = "../remote-externalities" } +remote-externalities = { path = "../remote-externalities" } diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 61869748c63ca..44f3b8e3b9cdf 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -20,10 +20,10 @@ use parity_scale_codec::{Decode, Encode}; use std::{fmt::Debug, str::FromStr}; use sc_service::Configuration; -use sc_cli::{CliConfiguration, ExecutionStrategy}; -use sc_executor::{WasmExecutionMethod, NativeExecutor}; -use sp_state_machine::StateMachine; +use sc_cli::{CliConfiguration, ExecutionStrategy, WasmExecutionMethod}; +use sc_executor::NativeExecutor; use sc_service::NativeExecutionDispatch; +use sp_state_machine::StateMachine; use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_core::storage::{StorageData, StorageKey, well_known_keys}; use frame_try_runtime::Target; @@ -45,6 +45,26 @@ pub struct TryRuntimeCmd { /// The state to use to run the migration. Should be a valid FILE or HTTP URI. #[structopt(short, long, default_value = "http://localhost:9933")] pub state: State, + + /// The execution strategy that should be used for benchmarks + #[structopt( + long = "execution", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = "Native", + )] + pub execution: ExecutionStrategy, + + /// Method for executing Wasm runtime code. + #[structopt( + long = "wasm-execution", + value_name = "METHOD", + possible_values = &WasmExecutionMethod::enabled_variants(), + case_insensitive = true, + default_value = "Interpreted" + )] + pub wasm_method: WasmExecutionMethod, } /// The state to use for a migration dry-run. @@ -80,19 +100,30 @@ impl TryRuntimeCmd { B: BlockT, ExecDispatch: NativeExecutionDispatch + 'static, { - let spec = config.chain_spec; let genesis_storage = spec.build_storage()?; - let code = StorageData(genesis_storage.top.get(well_known_keys::CODE).unwrap().to_vec()); + let code = StorageData( + genesis_storage + .top + .get(well_known_keys::CODE) + .expect("code key must exist in genesis storage; qed") + .to_vec(), + ); let code_key = StorageKey(well_known_keys::CODE.to_vec()); - let wasm_method = WasmExecutionMethod::Interpreted; - let strategy = ExecutionStrategy::Native; - let mut changes = Default::default(); + let wasm_method = self.wasm_method; + let execution = self.execution; - let heap_pages = Some(1024); - let executor = NativeExecutor::::new(wasm_method, heap_pages, 2); + let mut changes = Default::default(); + // don't really care about these -- use the default values. + let max_runtime_instances = config.max_runtime_instances; + let heap_pages = config.default_heap_pages; + let executor = NativeExecutor::::new( + wasm_method.into(), + heap_pages, + max_runtime_instances, + ); let ext = { use remote_externalities::{Builder, Mode, CacheConfig, OfflineConfig, OnlineConfig}; @@ -106,10 +137,11 @@ impl TryRuntimeCmd { })), }; + // inject the code into this ext. builder.inject(&[(code_key, code)]).build().await }; - let consumed_weight = StateMachine::<_, _, NumberFor, _>::new( + let encoded_result = StateMachine::<_, _, NumberFor, _>::new( &ext.backend, None, &mut changes, @@ -118,17 +150,19 @@ impl TryRuntimeCmd { &self.target.encode(), ext.extensions, &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) - .runtime_code() - .unwrap(), + .runtime_code()?, sp_core::testing::TaskExecutor::new(), ) - .execute(strategy.into()) - .unwrap(); + .execute(execution.into()) + .map_err(|e| format!("failed to execute 'TryRuntime_on_runtime_upgrade' due to {:?}", e))?; - let weight = ::decode(&mut &*consumed_weight).unwrap(); + let (weight, total_weight) = <(u64, u64) as Decode>::decode(&mut &*encoded_result) + .map_err(|e| format!("failed to decode output due to {:?}", e))?; log::info!( - "try-runtime executed without errors. Consumed weight = {}", + "try-runtime executed without errors. Consumed weight = {}, total weight = {} ({})", weight, + total_weight, + weight as f64 / total_weight as f64 ); Ok(()) diff --git a/utils/frame/try-runtime/remote-externalities/Cargo.toml b/utils/frame/try-runtime/remote-externalities/Cargo.toml index 5719aa67ad4b2..aca880f416806 100644 --- a/utils/frame/try-runtime/remote-externalities/Cargo.toml +++ b/utils/frame/try-runtime/remote-externalities/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "remote-externalities" -version = "3.0.0" +version = "0.9.0" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" diff --git a/utils/frame/try-runtime/remote-externalities/src/lib.rs b/utils/frame/try-runtime/remote-externalities/src/lib.rs index 8d5c19686f8d7..ab5a0b4e4cb36 100644 --- a/utils/frame/try-runtime/remote-externalities/src/lib.rs +++ b/utils/frame/try-runtime/remote-externalities/src/lib.rs @@ -105,11 +105,13 @@ use std::{ fs, path::{Path, PathBuf}, }; -use std::fmt::{Debug, Formatter, Result as FmtResult}; use log::*; use sp_core::{hashing::twox_128}; pub use sp_io::TestExternalities; -use sp_core::storage::{StorageKey, StorageData}; +use sp_core::{ + hexdisplay::HexDisplayExt, + storage::{StorageKey, StorageData}, +}; use futures::future::Future; type KeyPair = (StorageKey, StorageData); @@ -119,39 +121,6 @@ type Hash = sp_core::H256; const LOG_TARGET: &'static str = "remote-ext"; -/// Struct for better hex printing of slice types. -pub struct HexSlice<'a>(&'a [u8]); - -impl<'a> HexSlice<'a> { - pub fn new(data: &'a T) -> HexSlice<'a> - where - T: ?Sized + AsRef<[u8]> + 'a, - { - HexSlice(data.as_ref()) - } -} - -// You can choose to implement multiple traits, like Lower and UpperHex -impl Debug for HexSlice<'_> { - fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { - write!(f, "0x")?; - for byte in self.0 { - write!(f, "{:x}", byte)?; - } - Ok(()) - } -} -/// Extension trait for hex display. -pub trait HexDisplayExt { - fn hex_display(&self) -> HexSlice<'_>; -} - -impl> HexDisplayExt for T { - fn hex_display(&self) -> HexSlice<'_> { - HexSlice::new(self) - } -} - /// The execution mode. #[derive(Clone)] pub enum Mode { From b13ea317f2d33c973eb292cdd7447ffb58d8784c Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 18 Feb 2021 11:30:00 +0000 Subject: [PATCH 13/18] Fix build --- Cargo.lock | 6 +++--- bin/node/cli/Cargo.toml | 4 ++-- bin/node/runtime/Cargo.toml | 2 +- utils/frame/try-runtime/cli/Cargo.toml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67f7e049b95fd..08adc86d32564 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1863,7 +1863,7 @@ dependencies = [ [[package]] name = "frame-try-runtime" -version = "3.0.0" +version = "0.9.0" dependencies = [ "frame-support", "parity-scale-codec", @@ -6385,7 +6385,7 @@ dependencies = [ [[package]] name = "remote-externalities" -version = "3.0.0" +version = "0.9.0" dependencies = [ "async-std", "bincode", @@ -9928,7 +9928,7 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "try-runtime-cli" -version = "3.0.0" +version = "0.9.0" dependencies = [ "frame-try-runtime", "log", diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 1c821821c183f..65e1364298ba0 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -99,7 +99,7 @@ node-executor = { version = "2.0.0", path = "../executor" } sc-cli = { version = "0.9.0", optional = true, path = "../../../client/cli" } frame-benchmarking-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/benchmarking-cli" } node-inspect = { version = "0.8.0", optional = true, path = "../inspect" } -try-runtime-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/try-runtime/cli" } +try-runtime-cli = { version = "0.9.0", optional = true, path = "../../../utils/frame/try-runtime/cli" } # WASM-specific dependencies wasm-bindgen = { version = "0.2.57", optional = true } @@ -132,7 +132,7 @@ node-inspect = { version = "0.8.0", optional = true, path = "../inspect" } frame-benchmarking-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/benchmarking-cli" } substrate-build-script-utils = { version = "3.0.0", optional = true, path = "../../../utils/build-script-utils" } substrate-frame-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/frame-utilities-cli" } -try-runtime-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/try-runtime/cli" } +try-runtime-cli = { version = "0.9.0", optional = true, path = "../../../utils/frame/try-runtime/cli" } [build-dependencies.sc-cli] version = "0.9.0" diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index ebead89ff8b13..1a55efbf85157 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -43,7 +43,7 @@ frame-support = { version = "3.0.0", default-features = false, path = "../../../ frame-system = { version = "3.0.0", default-features = false, path = "../../../frame/system" } frame-system-benchmarking = { version = "3.0.0", default-features = false, path = "../../../frame/system/benchmarking", optional = true } frame-system-rpc-runtime-api = { version = "3.0.0", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } -frame-try-runtime = { version = "3.0.0", default-features = false, path = "../../../frame/try-runtime", optional = true } +frame-try-runtime = { version = "0.9.0", default-features = false, path = "../../../frame/try-runtime", optional = true } pallet-assets = { version = "3.0.0", default-features = false, path = "../../../frame/assets" } pallet-authority-discovery = { version = "3.0.0", default-features = false, path = "../../../frame/authority-discovery" } pallet-authorship = { version = "3.0.0", default-features = false, path = "../../../frame/authorship" } diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml index b539db5a3ebfa..73ae06d9eb7c7 100644 --- a/utils/frame/try-runtime/cli/Cargo.toml +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "try-runtime-cli" -version = "0.8.0" +version = "0.9.0" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -27,6 +27,6 @@ sp-blockchain = { version = "3.0.0", path = "../../../../primitives/blockchain" sp-runtime = { version = "3.0.0", path = "../../../../primitives/runtime" } sp-externalities = { version = "0.9.0", path = "../../../../primitives/externalities" } sp-core = { version = "3.0.0", path = "../../../../primitives/core" } -frame-try-runtime = { version = "3.0.0", path = "../../../../frame/try-runtime" } +frame-try-runtime = { version = "0.9.0", path = "../../../../frame/try-runtime" } remote-externalities = { path = "../remote-externalities" } From d3a2368167a3b25a0d08b6c64850a43e78c408c7 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 18 Feb 2021 12:24:31 +0000 Subject: [PATCH 14/18] Add some comments --- bin/node/cli/Cargo.toml | 2 ++ primitives/state-machine/src/testing.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 65e1364298ba0..7a765b0bde11d 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -163,6 +163,8 @@ runtime-benchmarks = [ "node-runtime/runtime-benchmarks", "frame-benchmarking-cli", ] +# Enable features that allow the runtime to be tried and debugged. Name might be subject to change +# in the near future. try-runtime = [ "node-runtime/try-runtime", "try-runtime-cli", diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 6ec27fc0a8d12..5f10fc0a276c4 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -49,7 +49,7 @@ where H::Out: codec::Codec + Ord, { /// The overlay changed storage. - pub overlay: OverlayedChanges, + overlay: OverlayedChanges, offchain_db: TestPersistentOffchainDB, storage_transaction_cache: StorageTransactionCache< as Backend>::Transaction, H, N>, From c8ba546b87b9d8c7ff01633723a74e36d8e5b9e0 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 18 Feb 2021 12:37:59 +0000 Subject: [PATCH 15/18] Remove allowing one pallet at a time. --- bin/node/runtime/src/lib.rs | 24 ++----------- frame/try-runtime/src/lib.rs | 49 +++----------------------- utils/frame/try-runtime/cli/src/lib.rs | 9 ++--- 3 files changed, 8 insertions(+), 74 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 34616b8d89960..c1ed6f6d5c5c2 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1334,29 +1334,9 @@ impl_runtime_apis! { #[cfg(feature = "try-runtime")] impl frame_try_runtime::TryRuntime for Runtime { - fn on_runtime_upgrade(target: frame_try_runtime::Target) -> Result<(Weight, Weight), sp_runtime::RuntimeString> { + fn on_runtime_upgrade() -> Result<(Weight, Weight), sp_runtime::RuntimeString> { frame_support::debug::RuntimeLogger::init(); - - let weight = match target { - frame_try_runtime::Target::All => { - frame_support::debug::info!("Dry-running all on-runtime-upgrades."); - Executive::try_runtime_upgrade()? - }, - frame_try_runtime::Target::Pallet(name) => { - let name = sp_std::str::from_utf8(&name).expect("pallet name is utf8-decodable; qed"); - frame_support::debug::info!("Dry-running on-runtime-upgrade of {}.", name); - - frame_try_runtime::match_pallet_on_runtime_upgrade!(name, - System, Utility, Babe, Timestamp, Authorship, Indices, Balances, - TransactionPayment, Staking, Session, Democracy, Council, - TechnicalCommittee, Elections, TechnicalMembership, Grandpa, Treasury, - Contracts, Sudo, ImOnline, AuthorityDiscovery, Offences, Historical, - RandomnessCollectiveFlip, Identity, Society, Recovery, Vesting, Scheduler, - Proxy, Multisig, Bounties, Tips, Assets, Mmr, Lottery, - ) - } - }; - + let weight = Executive::try_runtime_upgrade()?; Ok((weight, RuntimeBlockWeights::get().max_block)) } } diff --git a/frame/try-runtime/src/lib.rs b/frame/try-runtime/src/lib.rs index b47380348b478..dcd3a47878237 100644 --- a/frame/try-runtime/src/lib.rs +++ b/frame/try-runtime/src/lib.rs @@ -19,60 +19,19 @@ #![cfg_attr(not(feature = "std"), no_std)] -use codec::{Decode, Encode}; use sp_std::prelude::*; use frame_support::weights::Weight; -#[doc(hidden)] -pub use frame_support as _support; - -/// Helper macro to generate the match expression needed to match a pallet name to -/// its `on_runtime_upgrade()` implementation. -#[macro_export] -macro_rules! match_pallet_on_runtime_upgrade { - ($name:ident, $($pallet:ty),* $(,)*) => { - match $name { - $( - stringify!($pallet) => { - <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::pre_upgrade()?; - let weight = <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::on_runtime_upgrade(); - <$pallet as $crate::_support::traits::OnRuntimeUpgrade>::post_upgrade()?; - weight - }, - )* - _ => panic!("Unknown pallet name provided"), - } - }; -} - -/// Possible targets for try-runtime testing. -#[derive(Debug, Encode, Decode)] -pub enum Target { - /// All pallets. - All, - /// A single pallet. Inner value is the encoded pallet name. - Pallet(Vec), -} - -#[cfg(feature = "std")] -impl sp_std::str::FromStr for Target { - type Err = &'static str; - fn from_str(s: &str) -> Result { - Ok(if s.to_lowercase() == "all" { - Target::All - } else { - Target::Pallet(s.as_bytes().to_vec()) - }) - } -} - sp_api::decl_runtime_apis! { /// Runtime api for testing the execution of a runtime upgrade. pub trait TryRuntime { /// dry-run runtime upgrades, returning the total weight consumed. /// + /// This should do EXACTLY the same operations as the runtime would have done in the case of + /// a runtime upgrade (e.g. pallet ordering must be the same) + /// /// Returns the consumed weight of the migration in case of a successful one, combined with /// the total allowed block weight of the runtime. - fn on_runtime_upgrade(target: Target) -> Result<(Weight, Weight), sp_runtime::RuntimeString>; + fn on_runtime_upgrade() -> Result<(Weight, Weight), sp_runtime::RuntimeString>; } } diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 44f3b8e3b9cdf..92526379f471e 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -17,7 +17,7 @@ //! `Structopt`-ready struct for `try-runtime`. -use parity_scale_codec::{Decode, Encode}; +use parity_scale_codec::Decode; use std::{fmt::Debug, str::FromStr}; use sc_service::Configuration; use sc_cli::{CliConfiguration, ExecutionStrategy, WasmExecutionMethod}; @@ -26,7 +26,6 @@ use sc_service::NativeExecutionDispatch; use sp_state_machine::StateMachine; use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_core::storage::{StorageData, StorageKey, well_known_keys}; -use frame_try_runtime::Target; /// Various commands to try out the new runtime, over configurable states. /// @@ -38,10 +37,6 @@ pub struct TryRuntimeCmd { #[structopt(flatten)] pub shared_params: sc_cli::SharedParams, - /// The target pallet to run the migration against. - #[structopt(short, long, default_value = "All")] - pub target: Target, - /// The state to use to run the migration. Should be a valid FILE or HTTP URI. #[structopt(short, long, default_value = "http://localhost:9933")] pub state: State, @@ -147,7 +142,7 @@ impl TryRuntimeCmd { &mut changes, &executor, "TryRuntime_on_runtime_upgrade", - &self.target.encode(), + &[], ext.extensions, &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) .runtime_code()?, From 2f9ad0ecf6a246cfc6d58dc687884120b01c4d05 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 18 Feb 2021 17:59:13 +0000 Subject: [PATCH 16/18] More grumbles. --- primitives/core/src/hexdisplay.rs | 12 ------------ .../try-runtime/remote-externalities/src/lib.rs | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/primitives/core/src/hexdisplay.rs b/primitives/core/src/hexdisplay.rs index 211b9a97917f4..e590eec0e5aec 100644 --- a/primitives/core/src/hexdisplay.rs +++ b/primitives/core/src/hexdisplay.rs @@ -53,18 +53,6 @@ impl<'a> sp_std::fmt::Debug for HexDisplay<'a> { } } -/// Extension trait for wrapping a [`HexDisplay`]. -pub trait HexDisplayExt { - /// Display self as hex string. - fn hex_display(&self) -> HexDisplay<'_>; -} - -impl HexDisplayExt for T { - fn hex_display(&self) -> HexDisplay<'_> { - HexDisplay::from(self) - } -} - /// Simple trait to transform various types to `&[u8]` pub trait AsBytesRef { /// Transform `self` into `&[u8]`. diff --git a/utils/frame/try-runtime/remote-externalities/src/lib.rs b/utils/frame/try-runtime/remote-externalities/src/lib.rs index ab5a0b4e4cb36..6c8b49c7c85d5 100644 --- a/utils/frame/try-runtime/remote-externalities/src/lib.rs +++ b/utils/frame/try-runtime/remote-externalities/src/lib.rs @@ -109,7 +109,7 @@ use log::*; use sp_core::{hashing::twox_128}; pub use sp_io::TestExternalities; use sp_core::{ - hexdisplay::HexDisplayExt, + hexdisplay::HexDisplay, storage::{StorageKey, StorageData}, }; use futures::future::Future; @@ -302,7 +302,7 @@ impl Builder { "downloaded data for module {} (count: {} / prefix: {:?}).", f, module_kv.len(), - hashed_prefix.hex_display(), + HexDisplay::from(&hashed_prefix), ); filtered_kv.extend(module_kv); } From b8ab620fb7a5f50f01525ce2eeb6457bc568d56b Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 19 Feb 2021 11:31:17 +0000 Subject: [PATCH 17/18] relocate remote-ext --- Cargo.toml | 2 +- .../remote-externalities/Cargo.toml | 8 ++++---- .../remote-externalities/proxy_test | Bin .../remote-externalities/src/lib.rs | 0 utils/frame/try-runtime/cli/Cargo.toml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) rename utils/frame/{try-runtime => }/remote-externalities/Cargo.toml (73%) rename utils/frame/{try-runtime => }/remote-externalities/proxy_test (100%) rename utils/frame/{try-runtime => }/remote-externalities/src/lib.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 45816ab4163fa..adc8960ffd765 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -186,8 +186,8 @@ members = [ "utils/build-script-utils", "utils/fork-tree", "utils/frame/benchmarking-cli", + "utils/frame/remote-externalities", "utils/frame/frame-utilities-cli", - "utils/frame/try-runtime/remote-externalities", "utils/frame/try-runtime/cli", "utils/frame/rpc/support", "utils/frame/rpc/system", diff --git a/utils/frame/try-runtime/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml similarity index 73% rename from utils/frame/try-runtime/remote-externalities/Cargo.toml rename to utils/frame/remote-externalities/Cargo.toml index aca880f416806..41a3b26217860 100644 --- a/utils/frame/try-runtime/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -14,8 +14,8 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] jsonrpc-core-client = { version = "15.1.0", features = ["http"] } -sc-rpc-api = { version = "0.9.0", path = "../../../../client/rpc-api" } -sc-rpc = { version = "3.0.0", path = "../../../../client/rpc" } +sc-rpc-api = { version = "0.9.0", path = "../../../client/rpc-api" } +sc-rpc = { version = "3.0.0", path = "../../../client/rpc" } futures = "0.1.29" hex-literal = "0.3.1" @@ -24,8 +24,8 @@ log = "0.4.11" bincode = "1.3.1" tokio = "0.1.22" -sp-io = { version = "3.0.0", path = "../../../../primitives/io" } -sp-core = { version = "3.0.0", path = "../../../../primitives/core" } +sp-io = { version = "3.0.0", path = "../../../primitives/io" } +sp-core = { version = "3.0.0", path = "../../../primitives/core" } [dev-dependencies] async-std = { version = "1.6.5", features = ["attributes"] } diff --git a/utils/frame/try-runtime/remote-externalities/proxy_test b/utils/frame/remote-externalities/proxy_test similarity index 100% rename from utils/frame/try-runtime/remote-externalities/proxy_test rename to utils/frame/remote-externalities/proxy_test diff --git a/utils/frame/try-runtime/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs similarity index 100% rename from utils/frame/try-runtime/remote-externalities/src/lib.rs rename to utils/frame/remote-externalities/src/lib.rs diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml index 73ae06d9eb7c7..592d0a5b99d27 100644 --- a/utils/frame/try-runtime/cli/Cargo.toml +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -29,4 +29,4 @@ sp-externalities = { version = "0.9.0", path = "../../../../primitives/externali sp-core = { version = "3.0.0", path = "../../../../primitives/core" } frame-try-runtime = { version = "0.9.0", path = "../../../../frame/try-runtime" } -remote-externalities = { path = "../remote-externalities" } +remote-externalities = { path = "../../remote-externalities" } From b5e394b45aaa06bc5ece366cac113bad87b52b3e Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 19 Feb 2021 13:08:11 +0000 Subject: [PATCH 18/18] Fix build --- bin/node/cli/src/command.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 79fb2af860b43..d3689bdcd6743 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -153,14 +153,12 @@ pub fn run() -> Result<()> { Some(Subcommand::TryRuntime(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|config| { - use sc_service::TaskManager; // we don't need any of the components of new_partial, just a runtime, or a task // manager to do `async_run`. let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry); - let task_manager = TaskManager::new( + let task_manager = sc_service::TaskManager::new( config.task_executor.clone(), registry, - config.telemetry_span.clone(), ).unwrap(); Ok((cmd.run::(config), task_manager))