diff --git a/Cargo.lock b/Cargo.lock index 4b583c1c6a982..1ea460815c2f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7053,6 +7053,7 @@ dependencies = [ "pallet-election-provider-multi-phase", "pallet-election-provider-support-benchmarking", "pallet-elections-phragmen", + "pallet-example-mbm", "pallet-example-tasks", "pallet-fast-unstake", "pallet-glutton", @@ -9837,6 +9838,20 @@ dependencies = [ "sp-std 14.0.0", ] +[[package]] +name = "pallet-example-mbm" +version = "0.1.0" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "log", + "pallet-migrations", + "parity-scale-codec", + "scale-info", + "sp-io", +] + [[package]] name = "pallet-example-offchain-worker" version = "28.0.0" diff --git a/Cargo.toml b/Cargo.toml index e6162830375fd..067b65ff22993 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -331,6 +331,7 @@ members = [ "substrate/frame/examples/dev-mode", "substrate/frame/examples/frame-crate", "substrate/frame/examples/kitchensink", + "substrate/frame/examples/multi-block-migrations", "substrate/frame/examples/offchain-worker", "substrate/frame/examples/single-block-migrations", "substrate/frame/examples/split", diff --git a/prdoc/pr_2119.prdoc b/prdoc/pr_2119.prdoc new file mode 100644 index 0000000000000..d548b5af18a40 --- /dev/null +++ b/prdoc/pr_2119.prdoc @@ -0,0 +1,16 @@ +title: "Add example pallet for Multi-Block-Migrations" + +doc: + - audience: Runtime Dev + description: | + - Add an example pallet to demonstrate Multi-Block-Migrations. + - Add a `MigrationId` to frame-support for more convenient identification or migrations. + - Add default config prelude for testing in pallet-migrations. + +crates: + - name: frame-support + bump: minor + - name: pallet-migrations + bump: minor + - name: kitchensink-runtime + bump: patch diff --git a/substrate/bin/node/runtime/Cargo.toml b/substrate/bin/node/runtime/Cargo.toml index 8f68b1d3e2f09..f4ed75b919642 100644 --- a/substrate/bin/node/runtime/Cargo.toml +++ b/substrate/bin/node/runtime/Cargo.toml @@ -89,6 +89,7 @@ pallet-elections-phragmen = { path = "../../../frame/elections-phragmen", defaul pallet-example-tasks = { path = "../../../frame/examples/tasks", default-features = false } pallet-fast-unstake = { path = "../../../frame/fast-unstake", default-features = false } pallet-migrations = { path = "../../../frame/migrations", default-features = false } +pallet-example-mbm = { path = "../../../frame/examples/multi-block-migrations", default-features = false } pallet-nis = { path = "../../../frame/nis", default-features = false } pallet-grandpa = { path = "../../../frame/grandpa", default-features = false } pallet-im-online = { path = "../../../frame/im-online", default-features = false } @@ -188,6 +189,7 @@ std = [ "pallet-election-provider-multi-phase/std", "pallet-election-provider-support-benchmarking?/std", "pallet-elections-phragmen/std", + "pallet-example-mbm/std", "pallet-example-tasks/std", "pallet-fast-unstake/std", "pallet-glutton/std", @@ -294,6 +296,7 @@ runtime-benchmarks = [ "pallet-election-provider-multi-phase/runtime-benchmarks", "pallet-election-provider-support-benchmarking/runtime-benchmarks", "pallet-elections-phragmen/runtime-benchmarks", + "pallet-example-mbm/runtime-benchmarks", "pallet-example-tasks/runtime-benchmarks", "pallet-fast-unstake/runtime-benchmarks", "pallet-glutton/runtime-benchmarks", @@ -373,6 +376,7 @@ try-runtime = [ "pallet-democracy/try-runtime", "pallet-election-provider-multi-phase/try-runtime", "pallet-elections-phragmen/try-runtime", + "pallet-example-mbm/try-runtime", "pallet-example-tasks/try-runtime", "pallet-fast-unstake/try-runtime", "pallet-glutton/try-runtime", diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index a9606ac0bb759..191244c3bb060 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -320,6 +320,8 @@ impl pallet_example_tasks::Config for Runtime { type WeightInfo = pallet_example_tasks::weights::SubstrateWeight; } +impl pallet_example_mbm::Config for Runtime {} + impl pallet_utility::Config for Runtime { type RuntimeEvent = RuntimeEvent; type RuntimeCall = RuntimeCall; @@ -2439,6 +2441,9 @@ mod runtime { #[runtime::pallet_index(77)] pub type SkipFeelessPayment = pallet_skip_feeless_payment; + + #[runtime::pallet_index(78)] + pub type PalletExampleMbms = pallet_example_mbm; } /// The address format for describing accounts. @@ -2597,6 +2602,7 @@ mod benches { [pallet_whitelist, Whitelist] [pallet_tx_pause, TxPause] [pallet_safe_mode, SafeMode] + [pallet_example_mbm, PalletExampleMbms] ); } diff --git a/substrate/frame/examples/multi-block-migrations/Cargo.toml b/substrate/frame/examples/multi-block-migrations/Cargo.toml new file mode 100644 index 0000000000000..28eca8577154e --- /dev/null +++ b/substrate/frame/examples/multi-block-migrations/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "pallet-example-mbm" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +license = "Apache-2.0" +homepage = "https://substrate.io" +repository.workspace = true +description = "Example FRAME pallet for multi-block migrations" +publish = false + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.6.5", default-features = false } +pallet-migrations = { path = "../../migrations", default-features = false } +frame-support = { path = "../../support", default-features = false } +frame-system = { path = "../../system", default-features = false } +frame-benchmarking = { path = "../../benchmarking", default-features = false, optional = true } +log = { version = "0.4.20", default-features = false } +scale-info = { version = "2.10.0", default-features = false } +sp-io = { path = "../../../primitives/io", default-features = false } + +[features] +default = ["std"] +std = [ + "codec/std", + "frame-benchmarking?/std", + "frame-support/std", + "frame-system/std", + "log/std", + "pallet-migrations/std", + "scale-info/std", + "sp-io/std", +] +runtime-benchmarks = [ + "frame-benchmarking/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "pallet-migrations/runtime-benchmarks", +] +try-runtime = [ + "frame-support/try-runtime", + "frame-system/try-runtime", + "pallet-migrations/try-runtime", +] diff --git a/substrate/frame/examples/multi-block-migrations/src/lib.rs b/substrate/frame/examples/multi-block-migrations/src/lib.rs new file mode 100644 index 0000000000000..657c42b1662c8 --- /dev/null +++ b/substrate/frame/examples/multi-block-migrations/src/lib.rs @@ -0,0 +1,87 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg_attr(not(feature = "std"), no_std)] + +//! # Multi-Block Migrations Example Pallet +//! +//! This pallet serves as a minimal example of a pallet that uses the [Multi-Block Migrations +//! Framework](frame_support::migrations). You can observe how to configure it in a runtime in the +//! `kitchensink-runtime` crate. +//! +//! ## Introduction and Purpose +//! +//! The primary purpose of this pallet is to demonstrate the concept of Multi-Block Migrations in +//! Substrate. It showcases the migration of values from in the +//! [`MyMap`](`pallet::MyMap`) storage map a `u32` to a `u64` data type using the +//! [`SteppedMigration`](`frame_support::migrations::SteppedMigration`) implementation from the +//! [`migrations::v1`] module. +//! +//! The [`MyMap`](`pallet::MyMap`) storage item is defined in this `pallet`, and is +//! aliased to [`v0::MyMap`](`migrations::v1::v0::MyMap`) in the [`migrations::v1`] +//! module. +//! +//! ## How to Read the Documentation +//! +//! To access and navigate this documentation in your browser, use the following command: +//! +//! - `cargo doc --package pallet-example-mbm --open` +//! +//! This documentation is organized to help you understand the pallet's components, features, and +//! migration process. +//! +//! ## Example Usage +//! +//! To use this pallet and understand multi-block migrations, you can refer to the +//! [`migrations::v1`] module, which contains a step-by-step migration example. +//! +//! ## Pallet Structure +//! +//! The pallet is structured as follows: +//! +//! - [`migrations`]: Contains migration-related modules and migration logic. +//! - [`v1`](`migrations::v1`): Demonstrates the migration process for changing the data type in +//! the storage map. +//! - [`pallet`]: Defines the pallet configuration and storage items. +//! +//! ## Migration Safety +//! +//! When working with migrations, it's crucial to ensure the safety of your migrations. The +//! preferred tool to test migrations is +//! [`try-runtime-cli`](https://github.com/paritytech/try-runtime-cli). Support will be added to +//! dry-run MBMs once they are stable +//! (tracked: ). + +pub mod migrations; +mod mock; + +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use frame_support::{pallet_prelude::StorageMap, Blake2_128Concat}; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config {} + + /// Define a storage item to illustrate multi-block migrations. + #[pallet::storage] + pub type MyMap = StorageMap<_, Blake2_128Concat, u32, u64>; +} diff --git a/substrate/frame/examples/multi-block-migrations/src/migrations/mod.rs b/substrate/frame/examples/multi-block-migrations/src/migrations/mod.rs new file mode 100644 index 0000000000000..fa6e1f202cb2f --- /dev/null +++ b/substrate/frame/examples/multi-block-migrations/src/migrations/mod.rs @@ -0,0 +1,29 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// # Multi-Block Migrations Module +/// +/// This module showcases a simple use of the multi-block migrations framework. +pub mod v1; + +/// A unique identifier across all pallets. +/// +/// This constant represents a unique identifier for the migrations of this pallet. +/// It helps differentiate migrations for this pallet from those of others. Note that we don't +/// directly pull the crate name from the environment, since that would change if the crate were +/// ever to be renamed and could cause historic migrations to run again. +pub const PALLET_MIGRATIONS_ID: &[u8; 18] = b"pallet-example-mbm"; diff --git a/substrate/frame/examples/multi-block-migrations/src/migrations/v1/benchmarks.rs b/substrate/frame/examples/multi-block-migrations/src/migrations/v1/benchmarks.rs new file mode 100644 index 0000000000000..d14da2b705b6a --- /dev/null +++ b/substrate/frame/examples/multi-block-migrations/src/migrations/v1/benchmarks.rs @@ -0,0 +1,54 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Benchmark the multi-block-migration. + +#![cfg(feature = "runtime-benchmarks")] + +use crate::{ + migrations::{ + v1, + v1::{weights, weights::WeightInfo}, + }, + Config, Pallet, +}; +use frame_benchmarking::v2::*; +use frame_support::{migrations::SteppedMigration, weights::WeightMeter}; + +#[benchmarks] +mod benches { + use super::*; + + /// Benchmark a single step of the `v1::LazyMigrationV1` migration. + #[benchmark] + fn step() { + v1::v0::MyMap::::insert(0, 0); + let mut meter = WeightMeter::new(); + + #[block] + { + v1::LazyMigrationV1::>::step(None, &mut meter).unwrap(); + } + + // Check that the new storage is decodable: + assert_eq!(crate::MyMap::::get(0), Some(0)); + // uses twice the weight once for migration and then for checking if there is another key. + assert_eq!(meter.consumed(), weights::SubstrateWeight::::step() * 2); + } + + impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Runtime); +} diff --git a/substrate/frame/examples/multi-block-migrations/src/migrations/v1/mod.rs b/substrate/frame/examples/multi-block-migrations/src/migrations/v1/mod.rs new file mode 100644 index 0000000000000..2016b03de45e5 --- /dev/null +++ b/substrate/frame/examples/multi-block-migrations/src/migrations/v1/mod.rs @@ -0,0 +1,118 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Multi-Block Migration v1 +//! +//! This module showcases a simple migration that iterates over the values in the +//! [`v0::MyMap`](`crate::migrations::v1::v0::MyMap`) storage map, transforms them, +//! and inserts them into the [`MyMap`](`crate::pallet::MyMap`) storage map. + +use super::PALLET_MIGRATIONS_ID; +use crate::pallet::{Config, MyMap}; +use frame_support::{ + migrations::{MigrationId, SteppedMigration, SteppedMigrationError}, + pallet_prelude::PhantomData, + weights::WeightMeter, +}; + +mod benchmarks; +mod tests; +pub mod weights; + +/// Module containing the OLD (v0) storage items. +/// +/// Before running this migration, the storage alias defined here represents the +/// `on_chain` storage. +// This module is public only for the purposes of linking it in the documentation. It is not +// intended to be used by any other code. +pub mod v0 { + use super::Config; + use crate::pallet::Pallet; + use frame_support::{storage_alias, Blake2_128Concat}; + + #[storage_alias] + /// The storage item that is being migrated from. + pub type MyMap = StorageMap, Blake2_128Concat, u32, u32>; +} + +/// Migrates the items of the [`crate::MyMap`] map from `u32` to `u64`. +/// +/// The `step` function will be called once per block. It is very important that this function +/// *never* panics and never uses more weight than it got in its meter. The migrations should also +/// try to make maximal progress per step, so that the total time it takes to migrate stays low. +pub struct LazyMigrationV1(PhantomData<(T, W)>); +impl SteppedMigration for LazyMigrationV1 { + type Cursor = u32; + // Without the explicit length here the construction of the ID would not be infallible. + type Identifier = MigrationId<18>; + + /// The identifier of this migration. Which should be globally unique. + fn id() -> Self::Identifier { + MigrationId { pallet_id: *PALLET_MIGRATIONS_ID, version_from: 0, version_to: 1 } + } + + /// The actual logic of the migration. + /// + /// This function is called repeatedly until it returns `Ok(None)`, indicating that the + /// migration is complete. Ideally, the migration should be designed in such a way that each + /// step consumes as much weight as possible. However, this is simplified to perform one stored + /// value mutation per block. + fn step( + mut cursor: Option, + meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + let required = W::step(); + // If there is not enough weight for a single step, return an error. This case can be + // problematic if it is the first migration that ran in this block. But there is nothing + // that we can do about it here. + if meter.remaining().any_lt(required) { + return Err(SteppedMigrationError::InsufficientWeight { required }); + } + + // We loop here to do as much progress as possible per step. + loop { + if meter.try_consume(required).is_err() { + break; + } + + let mut iter = if let Some(last_key) = cursor { + // If a cursor is provided, start iterating from the stored value + // corresponding to the last key processed in the previous step. + // Note that this only works if the old and the new map use the same way to hash + // storage keys. + v0::MyMap::::iter_from(v0::MyMap::::hashed_key_for(last_key)) + } else { + // If no cursor is provided, start iterating from the beginning. + v0::MyMap::::iter() + }; + + // If there's a next item in the iterator, perform the migration. + if let Some((last_key, value)) = iter.next() { + // Migrate the inner value: u32 -> u64. + let value = value as u64; + // We can just insert here since the old and the new map share the same key-space. + // Otherwise it would have to invert the concat hash function and re-hash it. + MyMap::::insert(last_key, value); + cursor = Some(last_key) // Return the processed key as the new cursor. + } else { + cursor = None; // Signal that the migration is complete (no more items to process). + break + } + } + Ok(cursor) + } +} diff --git a/substrate/frame/examples/multi-block-migrations/src/migrations/v1/tests.rs b/substrate/frame/examples/multi-block-migrations/src/migrations/v1/tests.rs new file mode 100644 index 0000000000000..838ba29a62124 --- /dev/null +++ b/substrate/frame/examples/multi-block-migrations/src/migrations/v1/tests.rs @@ -0,0 +1,70 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(all(test, not(feature = "runtime-benchmarks")))] + +use crate::{ + migrations::{ + v1, + v1::{weights, weights::WeightInfo as _}, + }, + mock::{ + new_test_ext, run_to_block, AllPalletsWithSystem, MigratorServiceWeight, Runtime as T, + System, + }, +}; +use frame_support::traits::OnRuntimeUpgrade; +use pallet_migrations::WeightInfo as _; + +#[test] +fn lazy_migration_works() { + new_test_ext().execute_with(|| { + frame_support::__private::sp_tracing::try_init_simple(); + // Insert some values into the old storage map. + for i in 0..1024 { + v1::v0::MyMap::::insert(i, i); + } + + // Give it enough weight do do exactly 16 iterations: + let limit = ::WeightInfo::progress_mbms_none() + + pallet_migrations::Pallet::::exec_migration_max_weight() + + weights::SubstrateWeight::::step() * 16; + MigratorServiceWeight::set(&limit); + + System::set_block_number(1); + AllPalletsWithSystem::on_runtime_upgrade(); // onboard MBMs + + let mut last_decodable = 0; + for block in 2..=65 { + run_to_block(block); + let mut decodable = 0; + for i in 0..1024 { + if crate::MyMap::::get(i).is_some() { + decodable += 1; + } + } + + assert_eq!(decodable, last_decodable + 16); + last_decodable = decodable; + } + + // Check that everything is decodable now: + for i in 0..1024 { + assert_eq!(crate::MyMap::::get(i), Some(i as u64)); + } + }); +} diff --git a/substrate/frame/examples/multi-block-migrations/src/migrations/v1/weights.rs b/substrate/frame/examples/multi-block-migrations/src/migrations/v1/weights.rs new file mode 100644 index 0000000000000..6a5cf2ac59365 --- /dev/null +++ b/substrate/frame/examples/multi-block-migrations/src/migrations/v1/weights.rs @@ -0,0 +1,84 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for `pallet_example_mbm` +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 32.0.0 +//! DATE: 2024-03-26, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `Olivers-MBP`, CPU: `` +//! WASM-EXECUTION: `Compiled`, CHAIN: `None`, DB CACHE: `1024` + +// Executed Command: +// polkadot-omni-bencher +// v1 +// benchmark +// pallet +// --runtime +// target/release/wbuild/kitchensink-runtime/kitchensink_runtime.compact.compressed.wasm +// --pallet +// pallet_example_mbm +// --extrinsic +// +// --template +// substrate/.maintain/frame-weight-template.hbs +// --output +// substrate/frame/examples/multi-block-migrations/src/migrations/weights.rs + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use core::marker::PhantomData; + +/// Weight functions needed for `pallet_example_mbm`. +pub trait WeightInfo { + fn step() -> Weight; +} + +/// Weights for `pallet_example_mbm` using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + /// Storage: `PalletExampleMbms::MyMap` (r:2 w:1) + /// Proof: `PalletExampleMbms::MyMap` (`max_values`: None, `max_size`: Some(28), added: 2503, mode: `MaxEncodedLen`) + fn step() -> Weight { + // Proof Size summary in bytes: + // Measured: `28` + // Estimated: `5996` + // Minimum execution time: 6_000_000 picoseconds. + Weight::from_parts(8_000_000, 5996) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } +} + +// For backwards compatibility and tests. +impl WeightInfo for () { + /// Storage: `PalletExampleMbms::MyMap` (r:2 w:1) + /// Proof: `PalletExampleMbms::MyMap` (`max_values`: None, `max_size`: Some(28), added: 2503, mode: `MaxEncodedLen`) + fn step() -> Weight { + // Proof Size summary in bytes: + // Measured: `28` + // Estimated: `5996` + // Minimum execution time: 6_000_000 picoseconds. + Weight::from_parts(8_000_000, 5996) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } +} diff --git a/substrate/frame/examples/multi-block-migrations/src/mock.rs b/substrate/frame/examples/multi-block-migrations/src/mock.rs new file mode 100644 index 0000000000000..9da1d2051fa1f --- /dev/null +++ b/substrate/frame/examples/multi-block-migrations/src/mock.rs @@ -0,0 +1,93 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(test)] + +//! # Mock runtime for testing Multi-Block-Migrations +//! +//! This runtime is for testing only and should *never* be used in production. Please see the +//! comments on the specific config items. The core part of this runtime is the +//! [`pallet_migrations::Config`] implementation, where you define the migrations you want to run +//! using the [`Migrations`] type. + +use frame_support::{ + construct_runtime, derive_impl, + migrations::MultiStepMigrator, + pallet_prelude::Weight, + traits::{OnFinalize, OnInitialize}, +}; + +type Block = frame_system::mocking::MockBlock; + +impl crate::Config for Runtime {} + +frame_support::parameter_types! { + pub storage MigratorServiceWeight: Weight = Weight::from_parts(100, 100); // do not use in prod +} + +#[derive_impl(pallet_migrations::config_preludes::TestDefaultConfig)] +impl pallet_migrations::Config for Runtime { + // Here we inject the actual MBMs. Currently there is just one, but it accepts a tuple. + // + // # Example + // ```ignore + // type Migrations = (v1::Migration, v2::Migration, v3::Migration); + // ``` + #[cfg(not(feature = "runtime-benchmarks"))] + type Migrations = ( + crate::migrations::v1::LazyMigrationV1< + Runtime, + crate::migrations::v1::weights::SubstrateWeight, + >, + ); + #[cfg(feature = "runtime-benchmarks")] + type Migrations = pallet_migrations::mock_helpers::MockedMigrations; + type MaxServiceWeight = MigratorServiceWeight; +} + +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] +impl frame_system::Config for Runtime { + type Block = Block; + type MultiBlockMigrator = Migrator; +} + +// Construct the runtime using the `construct_runtime` macro, specifying the pallet_migrations. +construct_runtime! { + pub struct Runtime + { + System: frame_system, + Pallet: crate, + Migrator: pallet_migrations, + } +} + +pub fn new_test_ext() -> sp_io::TestExternalities { + sp_io::TestExternalities::new(Default::default()) +} + +#[allow(dead_code)] +pub fn run_to_block(n: u64) { + assert!(System::block_number() < n); + while System::block_number() < n { + let b = System::block_number(); + AllPalletsWithSystem::on_finalize(b); + // Done by Executive: + ::MultiBlockMigrator::step(); + System::set_block_number(b + 1); + AllPalletsWithSystem::on_initialize(b + 1); + } +} diff --git a/substrate/frame/migrations/src/lib.rs b/substrate/frame/migrations/src/lib.rs index 30ed9c08830c8..649bc314a12b5 100644 --- a/substrate/frame/migrations/src/lib.rs +++ b/substrate/frame/migrations/src/lib.rs @@ -276,9 +276,10 @@ pub mod pallet { #[pallet::pallet] pub struct Pallet(_); - #[pallet::config] + #[pallet::config(with_default)] pub trait Config: frame_system::Config { /// The overarching event type of the runtime. + #[pallet::no_default_bounds] type RuntimeEvent: From> + IsType<::RuntimeEvent>; /// All the multi-block migrations to run. @@ -286,12 +287,14 @@ pub mod pallet { /// Should only be updated in a runtime-upgrade once all the old migrations have completed. /// (Check that [`Cursor`] is `None`). #[cfg(not(feature = "runtime-benchmarks"))] + #[pallet::no_default] type Migrations: SteppedMigrations; /// Mocked migrations for benchmarking only. /// /// Should be configured to [`crate::mock_helpers::MockedMigrations`] in benchmarks. #[cfg(feature = "runtime-benchmarks")] + #[pallet::no_default] type Migrations: MockedMigrations; /// The maximal length of an encoded cursor. @@ -323,6 +326,45 @@ pub mod pallet { type WeightInfo: WeightInfo; } + /// Default implementations of [`DefaultConfig`], which can be used to implement [`Config`]. + pub mod config_preludes { + use super::{inject_runtime_type, DefaultConfig}; + use frame_support::{ + derive_impl, + migrations::FreezeChainOnFailedMigration, + pallet_prelude::{ConstU32, *}, + }; + use frame_system::limits::BlockWeights; + + /// Provides a viable default config that can be used with + /// [`derive_impl`](`frame_support::derive_impl`) to derive a testing pallet config + /// based on this one. + /// + /// See `Test` in the `default-config` example pallet's `test.rs` for an example of + /// a downstream user of this particular `TestDefaultConfig` + pub struct TestDefaultConfig; + + frame_support::parameter_types! { + /// Maximal weight per block that can be spent on migrations in tests. + pub TestMaxServiceWeight: Weight = <::BlockWeights as Get>::get().max_block.div(2); + } + + #[derive_impl(frame_system::config_preludes::TestDefaultConfig, no_aggregated_types)] + impl frame_system::DefaultConfig for TestDefaultConfig {} + + #[frame_support::register_default_impl(TestDefaultConfig)] + impl DefaultConfig for TestDefaultConfig { + #[inject_runtime_type] + type RuntimeEvent = (); + type CursorMaxLen = ConstU32<{ 1 << 16 }>; + type IdentifierMaxLen = ConstU32<{ 256 }>; + type MigrationStatusHandler = (); + type FailedMigrationHandler = FreezeChainOnFailedMigration; + type MaxServiceWeight = TestMaxServiceWeight; + type WeightInfo = (); + } + } + /// The currently active migration to run and its cursor. /// /// `None` indicates that no migration is running. @@ -421,12 +463,11 @@ pub mod pallet { } // The per-block service weight is sane. - #[cfg(not(test))] { let want = T::MaxServiceWeight::get(); let max = ::BlockWeights::get().max_block; - assert!(want.all_lte(max), "Service weight is larger than a block: {want} > {max}",); + assert!(want.all_lte(max), "Service weight is larger than a block: {want} > {max}"); } // Cursor MEL @@ -726,7 +767,8 @@ impl Pallet { } } - fn exec_migration_max_weight() -> Weight { + /// The maximal weight of calling the private `Self::exec_migration` function. + pub fn exec_migration_max_weight() -> Weight { T::WeightInfo::exec_migration_complete() .max(T::WeightInfo::exec_migration_completed()) .max(T::WeightInfo::exec_migration_skipped_historic()) diff --git a/substrate/frame/migrations/src/mock.rs b/substrate/frame/migrations/src/mock.rs index bcd6a189c5bf0..48ff175f81378 100644 --- a/substrate/frame/migrations/src/mock.rs +++ b/substrate/frame/migrations/src/mock.rs @@ -28,7 +28,7 @@ use frame_support::{ weights::Weight, }; use frame_system::EventRecord; -use sp_core::{ConstU32, H256}; +use sp_core::H256; type Block = frame_system::mocking::MockBlock; @@ -51,15 +51,14 @@ frame_support::parameter_types! { pub const MaxServiceWeight: Weight = Weight::MAX.div(10); } +#[derive_impl(crate::config_preludes::TestDefaultConfig)] impl crate::Config for Test { - type RuntimeEvent = RuntimeEvent; + #[cfg(feature = "runtime-benchmarks")] + type Migrations = crate::mock_helpers::MockedMigrations; + #[cfg(not(feature = "runtime-benchmarks"))] type Migrations = MockedMigrations; - type CursorMaxLen = ConstU32<65_536>; - type IdentifierMaxLen = ConstU32<256>; type MigrationStatusHandler = MockedMigrationStatusHandler; type FailedMigrationHandler = MockedFailedMigrationHandler; - type MaxServiceWeight = MaxServiceWeight; - type WeightInfo = (); } frame_support::parameter_types! { diff --git a/substrate/frame/migrations/src/mock_helpers.rs b/substrate/frame/migrations/src/mock_helpers.rs index 995ec0a922ccb..d230417d12e6c 100644 --- a/substrate/frame/migrations/src/mock_helpers.rs +++ b/substrate/frame/migrations/src/mock_helpers.rs @@ -79,7 +79,7 @@ impl SteppedMigrations for MockedMigrations { let mut count: u32 = cursor.as_ref().and_then(|c| Decode::decode(&mut &c[..]).ok()).unwrap_or(0); - log::debug!("MockedMigration: Step {}", count); + log::debug!("MockedMigration: Step {count} vs max {steps}"); if count != steps || matches!(kind, TimeoutAfter) { count += 1; return Some(Ok(Some(count.encode()))) diff --git a/substrate/frame/support/src/migrations.rs b/substrate/frame/support/src/migrations.rs index b8cbcd6904814..968639e02d35b 100644 --- a/substrate/frame/support/src/migrations.rs +++ b/substrate/frame/support/src/migrations.rs @@ -441,6 +441,16 @@ pub enum SteppedMigrationError { Failed, } +/// A generic migration identifier that can be used by MBMs. +/// +/// It is not required that migrations use this identifier type, but it can help. +#[derive(MaxEncodedLen, Encode, Decode)] +pub struct MigrationId { + pub pallet_id: [u8; N], + pub version_from: u8, + pub version_to: u8, +} + /// Notification handler for status updates regarding Multi-Block-Migrations. #[impl_trait_for_tuples::impl_for_tuples(8)] pub trait MigrationStatusHandler {