diff --git a/Cargo.lock b/Cargo.lock index 5584971260cd1..919343ed5b8e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2764,6 +2764,7 @@ dependencies = [ "sp-io", "sp-runtime", "sp-std", + "sp-storage", "sp-tracing", "sp-version", ] @@ -4077,6 +4078,7 @@ dependencies = [ "pallet-lottery", "pallet-membership", "pallet-message-queue", + "pallet-migrations", "pallet-mmr", "pallet-multisig", "pallet-nft-fractionalization", @@ -6795,6 +6797,30 @@ dependencies = [ "sp-weights", ] +[[package]] +name = "pallet-migrations" +version = "1.0.0" +dependencies = [ + "docify", + "frame-benchmarking", + "frame-executive", + "frame-support", + "frame-system", + "impl-trait-for-tuples", + "log", + "parity-scale-codec", + "pretty_assertions", + "scale-info", + "sp-api", + "sp-block-builder", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", + "sp-tracing", + "sp-version", +] + [[package]] name = "pallet-mmr" version = "4.0.0-dev" diff --git a/Cargo.toml b/Cargo.toml index ce93421bc9ff1..fa07df1d218d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -121,6 +121,7 @@ members = [ "frame/lottery", "frame/membership", "frame/merkle-mountain-range", + "frame/migrations", "frame/multisig", "frame/nicks", "frame/node-authorization", diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index 22fb01b62d0f0..f2dc1099fdbb4 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -392,6 +392,10 @@ impl_runtime_apis! { ) -> sp_inherents::CheckInherentsResult { data.check_extrinsics(&block) } + + fn after_inherents() -> sp_runtime::BlockAfterInherentsMode { + Executive::after_inherents() + } } impl sp_transaction_pool::runtime_api::TaggedTransactionQueue for Runtime { diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 5b7983baec0e4..9ed42667c2e76 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -85,6 +85,7 @@ pallet-identity = { version = "4.0.0-dev", default-features = false, path = "../ pallet-lottery = { version = "4.0.0-dev", default-features = false, path = "../../../frame/lottery" } pallet-membership = { version = "4.0.0-dev", default-features = false, path = "../../../frame/membership" } pallet-message-queue = { version = "7.0.0-dev", default-features = false, path = "../../../frame/message-queue" } +pallet-migrations = { version = "1.0.0", default-features = false, path = "../../../frame/migrations" } pallet-mmr = { version = "4.0.0-dev", default-features = false, path = "../../../frame/merkle-mountain-range" } pallet-multisig = { version = "4.0.0-dev", default-features = false, path = "../../../frame/multisig" } pallet-nfts = { version = "4.0.0-dev", default-features = false, path = "../../../frame/nfts" } @@ -171,6 +172,7 @@ std = [ "pallet-lottery/std", "pallet-membership/std", "pallet-message-queue/std", + "pallet-migrations/std", "pallet-mmr/std", "pallet-multisig/std", "pallet-nomination-pools/std", @@ -265,6 +267,7 @@ runtime-benchmarks = [ "pallet-lottery/runtime-benchmarks", "pallet-membership/runtime-benchmarks", "pallet-message-queue/runtime-benchmarks", + "pallet-migrations/runtime-benchmarks", "pallet-mmr/runtime-benchmarks", "pallet-multisig/runtime-benchmarks", "pallet-nomination-pools-benchmarking/runtime-benchmarks", @@ -328,6 +331,7 @@ try-runtime = [ "pallet-lottery/try-runtime", "pallet-membership/try-runtime", "pallet-message-queue/try-runtime", + "pallet-migrations/try-runtime", "pallet-mmr/try-runtime", "pallet-multisig/try-runtime", "pallet-nomination-pools/try-runtime", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index f59d1c96b8da1..c5d9ddacb0f31 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1860,6 +1860,35 @@ impl pallet_statement::Config for Runtime { type MaxAllowedBytes = MaxAllowedBytes; } +parameter_types! { + pub MbmServiceWeight: Weight = Perbill::from_percent(80) * RuntimeBlockWeights::get().max_block; + // FAIL-CI remove + pub Mbms: pallet_migrations::MigrationsOf = vec![ + Box::new(pallet_migrations::mock_helpers::MockedMigration( + pallet_migrations::mock_helpers::MockedMigrationKind::SucceedAfter, 0 + )), + Box::new(pallet_migrations::mock_helpers::MockedMigration( + pallet_migrations::mock_helpers::MockedMigrationKind::SucceedAfter, 2 + )), + Box::new(pallet_migrations::mock_helpers::MockedMigration( + pallet_migrations::mock_helpers::MockedMigrationKind::SucceedAfter, 3 + )), + Box::new(pallet_migrations::mock_helpers::MockedMigration( + pallet_migrations::mock_helpers::MockedMigrationKind::SucceedAfter, 20 + )) + ]; +} + +impl pallet_migrations::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + type Migrations = Mbms; + type Cursor = pallet_migrations::mock_helpers::MockedCursor; + type Identifier = pallet_migrations::mock_helpers::MockedIdentifier; + type OnMigrationUpdate = (); + type ServiceWeight = MbmServiceWeight; + type WeightInfo = pallet_migrations::weights::SubstrateWeight; +} + construct_runtime!( pub struct Runtime where Block = Block, @@ -1935,6 +1964,7 @@ construct_runtime!( MessageQueue: pallet_message_queue, Pov: frame_benchmarking_pallet_pov, Statement: pallet_statement, + MultiBlockMigrations: pallet_migrations, } ); @@ -1979,6 +2009,7 @@ pub type Executive = frame_executive::Executive< Runtime, AllPalletsWithSystem, Migrations, + MultiBlockMigrations, >; // All migrations executed on runtime upgrade as a nested tuple of types implementing @@ -2034,6 +2065,7 @@ mod benches { [pallet_lottery, Lottery] [pallet_membership, TechnicalMembership] [pallet_message_queue, MessageQueue] + [pallet_migrations, MultiBlockMigrations] [pallet_mmr, Mmr] [pallet_multisig, Multisig] [pallet_nomination_pools, NominationPoolsBench::] @@ -2111,6 +2143,10 @@ impl_runtime_apis! { fn check_inherents(block: Block, data: InherentData) -> CheckInherentsResult { data.check_extrinsics(&block) } + + fn after_inherents() -> sp_runtime::BlockAfterInherentsMode { + Executive::after_inherents() + } } impl sp_transaction_pool::runtime_api::TaggedTransactionQueue for Runtime { diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index 642900d2f35d8..c90926eba73bb 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -39,7 +39,7 @@ use sp_core::traits::SpawnNamed; use sp_inherents::InherentData; use sp_runtime::{ traits::{BlakeTwo256, Block as BlockT, Hash as HashT, Header as HeaderT}, - Digest, Percent, SaturatedConversion, + BlockAfterInherentsMode, Digest, Percent, SaturatedConversion, }; use std::{marker::PhantomData, pin::Pin, sync::Arc, time}; @@ -57,6 +57,8 @@ pub const DEFAULT_BLOCK_SIZE_LIMIT: usize = 4 * 1024 * 1024 + 512; const DEFAULT_SOFT_DEADLINE_PERCENT: Percent = Percent::from_percent(50); +const LOG_TARGET: &'static str = "sc-basic-authorship"; + /// [`Proposer`] factory. pub struct ProposerFactory { spawn_handle: Box, @@ -302,7 +304,7 @@ where .propose_with(inherent_data, inherent_digests, deadline, block_size_limit) .await; if tx.send(res).is_err() { - trace!("Could not send block production result to proposer!"); + trace!(target: LOG_TARGET, "Could not send block production result to proposer!"); } }), ); @@ -339,10 +341,36 @@ where block_size_limit: Option, ) -> Result, PR::Proof>, sp_blockchain::Error> { - let propose_with_start = time::Instant::now(); + let propose_with_timer = time::Instant::now(); let mut block_builder = self.client.new_block_at(self.parent_hash, inherent_digests, PR::ENABLED)?; + self.apply_inherents(&mut block_builder, inherent_data)?; + + let block_timer = time::Instant::now(); + let mode = block_builder.after_inherents()?; + let end_reason = match mode { + BlockAfterInherentsMode::ExtrinsicsAllowed => + self.apply_extrinsics(&mut block_builder, deadline, block_size_limit).await?, + BlockAfterInherentsMode::ExtrinsicsForbidden => EndProposingReason::ExtrinsicsForbidden, + }; + + let (block, storage_changes, proof) = block_builder.build()?.into_inner(); + let block_took = block_timer.elapsed(); + + let proof = + PR::into_proof(proof).map_err(|e| sp_blockchain::Error::Application(Box::new(e)))?; + + self.print_summary(&block, end_reason, block_took, propose_with_timer.elapsed()); + Ok(Proposal { block, proof, storage_changes }) + } + + /// Apply all inherents to the block. + fn apply_inherents( + &self, + block_builder: &mut sc_block_builder::BlockBuilder<'_, Block, C, B>, + inherent_data: InherentData, + ) -> Result<(), sp_blockchain::Error> { let create_inherents_start = time::Instant::now(); let inherents = block_builder.create_inherents(inherent_data)?; let create_inherents_end = time::Instant::now(); @@ -358,7 +386,7 @@ where for inherent in inherents { match block_builder.push(inherent) { Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() => { - warn!("⚠️ Dropping non-mandatory inherent from overweight block.") + warn!(target: LOG_TARGET, "⚠️ Dropping non-mandatory inherent from overweight block.") }, Err(ApplyExtrinsicFailed(Validity(e))) if e.was_mandatory() => { error!( @@ -367,12 +395,21 @@ where return Err(ApplyExtrinsicFailed(Validity(e))) }, Err(e) => { - warn!("❗️ Inherent extrinsic returned unexpected error: {}. Dropping.", e); + warn!(target: LOG_TARGET, "❗️ Inherent extrinsic returned unexpected error: {}. Dropping.", e); }, Ok(_) => {}, } } + Ok(()) + } + /// Apply as many extrinsics as possible to the block. + async fn apply_extrinsics( + &self, + block_builder: &mut sc_block_builder::BlockBuilder<'_, Block, C, B>, + deadline: time::Instant, + block_size_limit: Option, + ) -> Result { // proceed with transactions // We calculate soft deadline used only in case we start skipping transactions. let now = (self.now)(); @@ -380,7 +417,6 @@ where let left_micros: u64 = left.as_micros().saturated_into(); let soft_deadline = now + time::Duration::from_micros(self.soft_deadline_percent.mul_floor(left_micros)); - let block_timer = time::Instant::now(); let mut skipped = 0; let mut unqueue_invalid = Vec::new(); @@ -391,7 +427,7 @@ where let mut pending_iterator = select! { res = t1 => res, _ = t2 => { - log::warn!( + warn!(target: LOG_TARGET, "Timeout fired waiting for transaction pool at block #{}. \ Proceeding with production.", self.parent_number, @@ -402,8 +438,8 @@ where let block_size_limit = block_size_limit.unwrap_or(self.default_block_size_limit); - debug!("Attempting to push transactions from the pool."); - debug!("Pool status: {:?}", self.transaction_pool.status()); + debug!(target: LOG_TARGET, "Attempting to push transactions from the pool."); + debug!(target: LOG_TARGET, "Pool status: {:?}", self.transaction_pool.status()); let mut transaction_pushed = false; let end_reason = loop { @@ -415,7 +451,7 @@ where let now = (self.now)(); if now > deadline { - debug!( + debug!(target: LOG_TARGET, "Consensus deadline reached when pushing block transactions, \ proceeding with proposing." ); @@ -431,87 +467,109 @@ where pending_iterator.report_invalid(&pending_tx); if skipped < MAX_SKIPPED_TRANSACTIONS { skipped += 1; - debug!( + debug!(target: LOG_TARGET, "Transaction would overflow the block size limit, \ but will try {} more transactions before quitting.", MAX_SKIPPED_TRANSACTIONS - skipped, ); continue } else if now < soft_deadline { - debug!( + debug!(target: LOG_TARGET, "Transaction would overflow the block size limit, \ but we still have time before the soft deadline, so \ we will try a bit more." ); continue } else { - debug!("Reached block size limit, proceeding with proposing."); + debug!(target: LOG_TARGET, "Reached block size limit, proceeding with proposing."); break EndProposingReason::HitBlockSizeLimit } } - trace!("[{:?}] Pushing to the block.", pending_tx_hash); - match sc_block_builder::BlockBuilder::push(&mut block_builder, pending_tx_data) { + trace!(target: LOG_TARGET, "[{:?}] Pushing to the block.", pending_tx_hash); + match sc_block_builder::BlockBuilder::push(block_builder, pending_tx_data) { Ok(()) => { transaction_pushed = true; - debug!("[{:?}] Pushed to the block.", pending_tx_hash); + debug!(target: LOG_TARGET, "[{:?}] Pushed to the block.", pending_tx_hash); }, Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() => { pending_iterator.report_invalid(&pending_tx); if skipped < MAX_SKIPPED_TRANSACTIONS { skipped += 1; - debug!( + debug!(target: LOG_TARGET, "Block seems full, but will try {} more transactions before quitting.", MAX_SKIPPED_TRANSACTIONS - skipped, ); } else if (self.now)() < soft_deadline { - debug!( + debug!(target: LOG_TARGET, "Block seems full, but we still have time before the soft deadline, \ so we will try a bit more before quitting." ); } else { - debug!("Reached block weight limit, proceeding with proposing."); + debug!(target: LOG_TARGET, "Reached block weight limit, proceeding with proposing."); break EndProposingReason::HitBlockWeightLimit } }, Err(e) => { pending_iterator.report_invalid(&pending_tx); - debug!("[{:?}] Invalid transaction: {}", pending_tx_hash, e); + debug!(target: LOG_TARGET, "[{:?}] Invalid transaction: {}", pending_tx_hash, e); unqueue_invalid.push(pending_tx_hash); }, } }; if matches!(end_reason, EndProposingReason::HitBlockSizeLimit) && !transaction_pushed { - warn!( + warn!(target: LOG_TARGET, "Hit block size limit of `{}` without including any transaction!", block_size_limit, ); } self.transaction_pool.remove_invalid(&unqueue_invalid); + Ok(end_reason) + } - let (block, storage_changes, proof) = block_builder.build()?.into_inner(); - + /// Prints a summary and does telemetry + metrics. + fn print_summary( + &self, + block: &Block, + end_reason: EndProposingReason, + block_took: time::Duration, + propose_with_took: time::Duration, + ) { + let extrinsics = block.extrinsics(); self.metrics.report(|metrics| { - metrics.number_of_transactions.set(block.extrinsics().len() as u64); - metrics.block_constructed.observe(block_timer.elapsed().as_secs_f64()); - + metrics.number_of_transactions.set(extrinsics.len() as u64); + metrics.block_constructed.observe(block_took.as_secs_f64()); metrics.report_end_proposing_reason(end_reason); + metrics.create_block_proposal_time.observe(propose_with_took.as_secs_f64()); }); + let extrinsics_summary = if extrinsics.is_empty() { + if end_reason == EndProposingReason::ExtrinsicsForbidden { + "extrinsics forbidden" + } else { + "no extrinsics" + } + .to_string() + } else { + format!( + "extrinsics ({}): [{}]", + extrinsics.len(), + extrinsics + .iter() + .map(|xt| BlakeTwo256::hash_of(xt).to_string()) + .collect::>() + .join(", ") + ) + }; + info!( - "🎁 Prepared block for proposing at {} ({} ms) [hash: {:?}; parent_hash: {}; extrinsics ({}): [{}]]", + "🎁 Prepared block for proposing at {} ({} ms) [hash: {:?}; parent_hash: {}; {extrinsics_summary}", block.header().number(), - block_timer.elapsed().as_millis(), + block_took.as_millis(), ::Hash::from(block.header().hash()), block.header().parent_hash(), - block.extrinsics().len(), - block.extrinsics() - .iter() - .map(|xt| BlakeTwo256::hash_of(xt).to_string()) - .collect::>() - .join(", ") ); telemetry!( self.telemetry; @@ -520,18 +578,6 @@ where "number" => ?block.header().number(), "hash" => ?::Hash::from(block.header().hash()), ); - - let proof = - PR::into_proof(proof).map_err(|e| sp_blockchain::Error::Application(Box::new(e)))?; - - let propose_with_end = time::Instant::now(); - self.metrics.report(|metrics| { - metrics.create_block_proposal_time.observe( - propose_with_end.saturating_duration_since(propose_with_start).as_secs_f64(), - ); - }); - - Ok(Proposal { block, proof, storage_changes }) } } diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index f055d4688822a..37d290f3220c6 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -36,7 +36,7 @@ use sp_core::ExecutionContext; use sp_runtime::{ legacy, traits::{Block as BlockT, Hash, HashFor, Header as HeaderT, NumberFor, One}, - Digest, + BlockAfterInherentsMode, Digest, }; use sc_client_api::backend; @@ -198,6 +198,14 @@ where }) } + /// Called after inherents but before extrinsics have been applied. + pub fn after_inherents(&self) -> Result { + // FAIL-CI why 'with_context'?! + self.api + .after_inherents_with_context(self.parent_hash, ExecutionContext::BlockConstruction) + .map_err(Into::into) + } + /// Push onto the block's list of extrinsics. /// /// This will ensure the extrinsic can be validly executed (by executing it). diff --git a/client/proposer-metrics/src/lib.rs b/client/proposer-metrics/src/lib.rs index 012e8ca769a96..a9aa5a0e0e302 100644 --- a/client/proposer-metrics/src/lib.rs +++ b/client/proposer-metrics/src/lib.rs @@ -44,11 +44,14 @@ impl MetricsLink { } /// The reason why proposing a block ended. +#[derive(Clone, Copy, PartialEq, Eq)] pub enum EndProposingReason { NoMoreTransactions, HitDeadline, HitBlockSizeLimit, HitBlockWeightLimit, + /// No extrinsics are allowed in the block. + ExtrinsicsForbidden, } /// Authorship metrics. @@ -112,6 +115,7 @@ impl Metrics { EndProposingReason::NoMoreTransactions => "no_more_transactions", EndProposingReason::HitBlockSizeLimit => "hit_block_size_limit", EndProposingReason::HitBlockWeightLimit => "hit_block_weight_limit", + EndProposingReason::ExtrinsicsForbidden => "extrinsics_forbidden", }; self.end_proposing_reason.with_label_values(&[reason]).inc(); diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml index 2532df31682f7..e4afa157b0533 100644 --- a/frame/executive/Cargo.toml +++ b/frame/executive/Cargo.toml @@ -24,6 +24,7 @@ sp-core = { version = "21.0.0", default-features = false, path = "../../primitiv sp-io = { version = "23.0.0", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "24.0.0", default-features = false, path = "../../primitives/runtime" } sp-std = { version = "8.0.0", default-features = false, path = "../../primitives/std" } +sp-storage = { version = "13.0.0", default-features = false, path = "../../primitives/storage" } sp-tracing = { version = "10.0.0", default-features = false, path = "../../primitives/tracing" } [dev-dependencies] @@ -48,6 +49,7 @@ std = [ "sp-io/std", "sp-runtime/std", "sp-std/std", + "sp-storage/std", "sp-tracing/std", ] try-runtime = ["frame-support/try-runtime", "frame-try-runtime/try-runtime", "sp-runtime/try-runtime"] diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 31cbb0ee7ba0d..751ca871dfbb8 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -133,7 +133,7 @@ use sp_runtime::{ ValidateUnsigned, Zero, }, transaction_validity::{TransactionSource, TransactionValidity}, - ApplyExtrinsicResult, + ApplyExtrinsicResult, BlockAfterInherentsMode, }; use sp_std::{marker::PhantomData, prelude::*}; @@ -165,6 +165,7 @@ pub struct Executive< UnsignedValidator, AllPalletsWithSystem, OnRuntimeUpgrade = (), + MultiStepMigrator = (), >( PhantomData<( System, @@ -173,6 +174,7 @@ pub struct Executive< UnsignedValidator, AllPalletsWithSystem, OnRuntimeUpgrade, + MultiStepMigrator, )>, ); @@ -187,9 +189,17 @@ impl< + OnFinalize + OffchainWorker, COnRuntimeUpgrade: OnRuntimeUpgrade, + MultiStepMigrator: frame_support::migrations::MultiStepMigrator, > ExecuteBlock - for Executive -where + for Executive< + System, + Block, + Context, + UnsignedValidator, + AllPalletsWithSystem, + COnRuntimeUpgrade, + MultiStepMigrator, + > where Block::Extrinsic: Checkable + Codec, CheckedOf: Applyable + GetDispatchInfo, CallOf: @@ -205,6 +215,7 @@ where UnsignedValidator, AllPalletsWithSystem, COnRuntimeUpgrade, + MultiStepMigrator, >::execute_block(block); } } @@ -222,8 +233,17 @@ impl< + OffchainWorker + frame_support::traits::TryState, COnRuntimeUpgrade: OnRuntimeUpgrade, - > Executive -where + MultiStepMigrator: frame_support::migrations::MultiStepMigrator, + > + Executive< + System, + Block, + Context, + UnsignedValidator, + AllPalletsWithSystem, + COnRuntimeUpgrade, + MultiStepMigrator, + > where Block::Extrinsic: Checkable + Codec, CheckedOf: Applyable + GetDispatchInfo, CallOf: @@ -293,7 +313,12 @@ where // post-extrinsics book-keeping >::note_finished_extrinsics(); - Self::idle_and_finalize_hook(*header.number()); + let is_upgrading = MultiStepMigrator::is_upgrading(); // FAIL-CI fix this properly + if !is_upgrading { + Self::on_idle_hook(*header.number()); + } + + Self::on_finalize_hook(*header.number()); // run the try-state checks of all pallets, ensuring they don't alter any state. let _guard = frame_support::StorageNoopGuard::default(); @@ -378,8 +403,17 @@ impl< + OnFinalize + OffchainWorker, COnRuntimeUpgrade: OnRuntimeUpgrade, - > Executive -where + MultiStepMigrator: frame_support::migrations::MultiStepMigrator, + > + Executive< + System, + Block, + Context, + UnsignedValidator, + AllPalletsWithSystem, + COnRuntimeUpgrade, + MultiStepMigrator, + > where Block::Extrinsic: Checkable + Codec, CheckedOf: Applyable + GetDispatchInfo, CallOf: @@ -454,7 +488,8 @@ where } } - fn initial_checks(block: &Block) { + /// Returns the index of the first extrinsic in the block. + fn initial_checks(block: &Block) -> u32 { sp_tracing::enter_span!(sp_tracing::Level::TRACE, "initial_checks"); let header = block.header(); @@ -467,8 +502,9 @@ where "Parent hash should be valid.", ); - if let Err(i) = System::ensure_inherents_are_first(block) { - panic!("Invalid inherent position for extrinsic at index {}", i); + match System::ensure_inherents_are_first(block) { + Ok(first_extrinsic_index) => first_extrinsic_index, + Err(i) => panic!("Invalid inherent position for extrinsic at index {}", i), } } @@ -477,53 +513,90 @@ where sp_io::init_tracing(); sp_tracing::within_span! { sp_tracing::info_span!("execute_block", ?block); - + // `on_runtime_upgrade` and `on_initialize`. Self::initialize_block(block.header()); - // any initial checks - Self::initial_checks(&block); + // Check the block and panic if invalid. + let first_extrinsic_index = Self::initial_checks(&block); + let (header, dispatchables) = block.deconstruct(); + + // Process inherents (if any). + let num_inherents = first_extrinsic_index as usize; + Self::execute_dispatchables(dispatchables.iter().take(num_inherents)); - // execute extrinsics - let (header, extrinsics) = block.deconstruct(); - Self::execute_extrinsics_with_book_keeping(extrinsics, *header.number()); + match Self::after_inherents() { + BlockAfterInherentsMode::ExtrinsicsForbidden => { + if num_inherents < dispatchables.len() { + panic!("Extrinsics are not allowed"); + } + }, + BlockAfterInherentsMode::ExtrinsicsAllowed => { + Self::execute_dispatchables(dispatchables.iter().skip(num_inherents)); + }, + } + // Dispatchable processing is done now. + >::note_finished_extrinsics(); + // TODO this could be optimized to be one less storage read. + if !MultiStepMigrator::is_upgrading() { + Self::on_idle_hook(*header.number()); + } + + Self::on_finalize_hook(*header.number()); // any final checks Self::final_checks(&header); } } - /// Execute given extrinsics and take care of post-extrinsics book-keeping. - fn execute_extrinsics_with_book_keeping( - extrinsics: Vec, - block_number: NumberFor, - ) { - extrinsics.into_iter().for_each(|e| { - if let Err(e) = Self::apply_extrinsic(e) { + /// Progress ongoing MBM migrations. + // Used by the block builder and Executive. + pub fn after_inherents() -> BlockAfterInherentsMode { + let is_upgrading = MultiStepMigrator::is_upgrading(); + if is_upgrading { + let used_weight = MultiStepMigrator::step(); + >::register_extra_weight_unchecked( + used_weight, + DispatchClass::Mandatory, + ); + } + + // TODO `poll` hook goes here. + + if is_upgrading { + BlockAfterInherentsMode::ExtrinsicsForbidden + } else { + BlockAfterInherentsMode::ExtrinsicsAllowed + } + } + + /// Execute given extrinsics. + fn execute_dispatchables<'a>(dispatchables: impl Iterator) { + dispatchables.into_iter().for_each(|e| { + if let Err(e) = Self::apply_extrinsic(e.clone()) { let err: &'static str = e.into(); panic!("{}", err) } }); - - // post-extrinsics book-keeping - >::note_finished_extrinsics(); - - Self::idle_and_finalize_hook(block_number); } /// Finalize the block - it is up the caller to ensure that all header fields are valid /// except state-root. + // Note: Only used by the block builder - not Executive itself. pub fn finalize_block() -> System::Header { sp_io::init_tracing(); sp_tracing::enter_span!(sp_tracing::Level::TRACE, "finalize_block"); >::note_finished_extrinsics(); let block_number = >::block_number(); - Self::idle_and_finalize_hook(block_number); + if !MultiStepMigrator::is_upgrading() { + Self::on_idle_hook(block_number); + } + Self::on_finalize_hook(block_number); >::finalize() } - fn idle_and_finalize_hook(block_number: NumberFor) { + fn on_idle_hook(block_number: NumberFor) { let weight = >::block_weight(); let max_weight = >::get().max_block; let remaining_weight = max_weight.saturating_sub(weight.total()); @@ -538,7 +611,9 @@ where DispatchClass::Mandatory, ); } + } + fn on_finalize_hook(block_number: NumberFor) { >::on_finalize(block_number); } @@ -564,6 +639,11 @@ where // Decode parameters and dispatch let dispatch_info = xt.get_dispatch_info(); + if dispatch_info.class != DispatchClass::Mandatory && MultiStepMigrator::is_upgrading() { + // The block builder respects this by using the mode returned by `after_inherents`. + panic!("Only Mandatory extrinsics are allowed during Multi-Block-Migrations"); + } + // Check whether we need to error because extrinsics are paused. let r = Applyable::apply::(xt, &dispatch_info, encoded_len)?; // Mandatory(inherents) are not allowed to fail. diff --git a/frame/migrations/Cargo.toml b/frame/migrations/Cargo.toml new file mode 100644 index 0000000000000..bc3e67ff1efbe --- /dev/null +++ b/frame/migrations/Cargo.toml @@ -0,0 +1,63 @@ +[package] +name = "pallet-migrations" +version = "1.0.0" +description = "FRAME pallet to execute multi-block migrations." +authors = ["Parity Technologies "] +homepage = "https://substrate.io" +edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/paritytech/substrate" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive"] } +docify = "0.1.14" +impl-trait-for-tuples = "0.2.2" +log = "0.4.18" +scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } + +frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../benchmarking" } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +sp-core = { version = "21.0.0", path = "../../primitives/core", default-features = false } +sp-std = { version = "8.0.0", path = "../../primitives/std", default-features = false } +sp-runtime = { version = "24.0.0", path = "../../primitives/runtime", default-features = false } + +[dev-dependencies] +frame-executive = { version = "4.0.0-dev", path = "../executive" } +sp-api = { version = "4.0.0-dev", path = "../../primitives/api", features = [ "std" ] } +sp-block-builder = { version = "4.0.0-dev", path = "../../primitives/block-builder", features = [ "std" ] } +sp-io = { version = "23.0.0", path = "../../primitives/io", features = [ "std" ] } +sp-tracing = { version = "10.0.0", path = "../../primitives/tracing", features = [ "std" ] } +sp-version = { version = "22.0.0", path = "../../primitives/version", features = [ "std" ] } + +docify = "0.1.14" +pretty_assertions = "1.3.0" + +[features] +default = ["std"] + +std = [ + "codec/std", + "frame-benchmarking?/std", + "frame-support/std", + "frame-system/std", + "scale-info/std", + "sp-core/std", + "sp-std/std", + "sp-runtime/std" +] + +runtime-benchmarks = [ + "frame-benchmarking/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "sp-runtime/runtime-benchmarks" +] + +try-runtime = [ + "frame-support/try-runtime", + "sp-runtime/try-runtime" +] diff --git a/frame/migrations/src/benchmarking.rs b/frame/migrations/src/benchmarking.rs new file mode 100644 index 0000000000000..ca14c774e94c3 --- /dev/null +++ b/frame/migrations/src/benchmarking.rs @@ -0,0 +1,108 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(feature = "runtime-benchmarks")] + +use super::*; + +use frame_benchmarking::v2::*; +use frame_system::{Pallet as System, RawOrigin}; + +#[benchmarks] +mod benches { + use super::*; + use frame_support::traits::Hooks; + + #[benchmark] + fn on_runtime_upgrade() { + assert!(!Cursor::::exists()); + + #[block] + { + Pallet::::on_runtime_upgrade(); + } + } + + #[benchmark] + fn on_init_fast_path() { + Cursor::::set(Some(cursor::())); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::on_initialize(1u32.into()); + } + } + + #[benchmark] + fn on_init_base() { + // FAIL-CI + Cursor::::set(Some(cursor::())); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::on_initialize(1u32.into()); + } + } + + #[benchmark] + fn on_init_loop() { + System::::set_block_number(1u32.into()); + Pallet::::on_runtime_upgrade(); + + #[block] + { + Pallet::::on_initialize(1u32.into()); + } + } + + /// Benchmarks the slowest path of `change_value`. + #[benchmark] + fn force_set_cursor() { + Cursor::::set(Some(cursor::())); + + #[extrinsic_call] + _(RawOrigin::Root, Some(cursor::())); + } + + #[benchmark] + fn clear_historic(n: Linear<0, 1000>) { + //for i in 0..n { // TODO + // Historic::::insert(i.into(), ()); + //} + + #[extrinsic_call] + _(RawOrigin::Root, None, None); + } + + fn cursor() -> MigrationCursor { + // Note: The weight of a function can depend on the weight of reading the `inner_cursor`. + // `Cursor` is a user provided type. Now instead of requiring something like `Cursor: + // From`, we instead rely on the fact that it is MEL and the PoV benchmarking will + // therefore already take the MEL bound, even when the cursor in storage is `None`. + MigrationCursor::Active(ActiveCursor { + index: u32::MAX, + inner_cursor: None, + started_at: 0u32.into(), + }) + } + + // Implements a test for each benchmark. Execute with: + // `cargo test -p pallet-migrations --features runtime-benchmarks`. + impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/frame/migrations/src/lib.rs b/frame/migrations/src/lib.rs new file mode 100644 index 0000000000000..0ca69f2debc44 --- /dev/null +++ b/frame/migrations/src/lib.rs @@ -0,0 +1,515 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![deny(missing_docs)] +#![deny(rustdoc::broken_intra_doc_links)] + +//! # `pallet-migrations` +//! +//! Provides multi block migrations for FRAME runtimes. +//! +//! ## Overview +//! +//! The pallet takes care of executing a batch of multi-step migrations over multiple blocks. The +//! process starts on each runtime upgrade. Normal and operational transactions are paused while +//! migrations are on-going. +//! +//! ### Example +//! +//! This example demonstrates a simple mocked walk through of a basic success scenario. The pallet +//! is configured with two migrations: one succeeding after just one step, and the second one +//! succeeding after two steps. A runtime upgrade is then enacted and the block number is advanced +//! until all migrations finish executing. Afterwards, the recorded historic migrations are +//! checked and events are asserted. +#![doc = docify::embed!("frame/migrations/src/tests.rs", simple_works)] +//! +//! ## Pallet API +//! +//! See the [`pallet`] module for more information about the interfaces this pallet exposes, +//! including its configuration trait, dispatchables, storage items, events and errors. +//! +//! Otherwise noteworthy API of this pallet include its implementation of the +//! [`MultiStepMigrator`] trait. This can be plugged into `frame-executive` to check for +//! transaction suspension. +//! +//! ### Design Goals +//! +//! 1. Must automatically execute migrations over multiple blocks. +//! 2. Must prevent other (non-mandatory) transactions to execute in the meantime. +//! 3. Must respect pessimistic weight bounds of migrations. +//! 4. Must execute migrations in order. Skipping is not allowed; migrations are run on an +//! all-or-nothing basis. 5. Must prevent re-execution of migrations. +//! 6. Must provide transactional storage semantics for migrations. +//! 7. Must guarantee progress. +//! +//! ### Design +//! +//! Migrations are provided to the pallet through the associated type [`Config::Migrations`] of type +//! `Get { + /// Points to the currently active migration and its cursor. + Active(ActiveCursor), + /// Migration got stuck and cannot proceed. + Stuck, +} + +impl MigrationCursor { + /// Maybe return self as an `ActiveCursor`. + pub fn as_active(&self) -> Option<&ActiveCursor> { + match self { + MigrationCursor::Active(active) => Some(active), + MigrationCursor::Stuck => None, + } + } +} + +impl From> + for MigrationCursor +{ + fn from(active: ActiveCursor) -> Self { + MigrationCursor::Active(active) + } +} + +/// Points to the currently active migration and its inner cursor. +#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode, scale_info::TypeInfo, MaxEncodedLen)] +pub struct ActiveCursor { + index: u32, + inner_cursor: Option, + started_at: BlockNumber, +} + +impl ActiveCursor { + /// Advance the cursor to the next migration. + pub(crate) fn advance(&mut self, current_block: BlockNumber) { + self.index.saturating_inc(); + self.inner_cursor = None; + self.started_at = current_block; + } +} + +/// Convenience alias for [`MigrationCursor`]. +pub type CursorOf = + MigrationCursor<::Cursor, ::BlockNumber>; + +/// Convenience alias for [`ActiveCursor`]. +pub type ActiveCursorOf = + ActiveCursor<::Cursor, ::BlockNumber>; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config { + /// The overarching event type of the runtime. + type RuntimeEvent: From> + IsType<::RuntimeEvent>; + + /// All the multi-block migrations to run. + /// + /// Should only be updated in a runtime-upgrade once all the old migrations have completed. + /// (Check that `Cursor` is `None`). + type Migrations: SteppedMigrations; + + /// The cursor type that is shared across all migrations. + type Cursor: FullCodec + MaxEncodedLen + TypeInfo + Parameter; + + /// The identifier type that is shared across all migrations. + type Identifier: FullCodec + MaxEncodedLen + TypeInfo; + + /// Notifications for status updates of a runtime upgrade. + /// + /// Can be used to pause XCM etc. + type OnMigrationUpdate: OnMigrationUpdate; + + /// The weight to spend each block to execute migrations. + type ServiceWeight: Get; + + /// Weight information for the calls and functions of this pallet. + type WeightInfo: WeightInfo; + } + + /// The currently active migration to run and its cursor. + /// + /// `None` indicates that no migration is running. + #[pallet::storage] + pub type Cursor = StorageValue<_, CursorOf, OptionQuery>; + + /// Set of all successfully executed migrations. + /// + /// This is used as blacklist, to not re-execute migrations that have not been removed from the + /// codebase yet. Governance can regularly clear this out via `clear_historic`. + #[pallet::storage] + pub type Historic = StorageMap<_, Twox64Concat, T::Identifier, (), OptionQuery>; + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// A Runtime upgrade started. + /// + /// Its end is indicated by `UpgradeCompleted` or `UpgradeFailed`. + UpgradeStarted { + /// The number of migrations that this upgrade contains. + /// + /// This can be used to design a progress indicator in combination with counting the + /// `MigrationCompleted` and `MigrationSkippedHistoric` events. + migrations: u32, + }, + /// The current runtime upgrade completed. + /// + /// This implies that all of its migrations completed successfully as well. + UpgradeCompleted, + /// Runtime upgrade failed. + /// + /// This is very bad and will require governance intervention. + UpgradeFailed, + /// A migration was skipped since it was already executed in the past. + MigrationSkippedHistoric { + /// The index of the skipped migration within the [`Config::Migrations`] list. + index: u32, + }, + /// A migration progressed. + MigrationAdvanced { + /// The index of the migration within the [`Config::Migrations`] list. + index: u32, + /// The number of blocks that elapsed since the migration started. + blocks: T::BlockNumber, + }, + /// A Migration completed. + MigrationCompleted { + /// The index of the migration within the [`Config::Migrations`] list. + index: u32, + /// The number of blocks that elapsed since the migration started. + blocks: T::BlockNumber, + }, + /// A Migration failed. + /// + /// This implies that the whole upgrade failed and governance intervention is required. + MigrationFailed { + /// The index of the migration within the [`Config::Migrations`] list. + index: u32, + /// The number of blocks that elapsed since the migration started. + blocks: T::BlockNumber, + }, + /// The set of historical migrations has been cleared. + HistoricCleared { + /// Should be passed to `clear_historic` in a successive call. + next_cursor: Option>, + }, + } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_runtime_upgrade() -> Weight { + Self::onboard_new_mbms() + } + } + + #[pallet::call(weight = T::WeightInfo)] + impl Pallet { + /// Allows root to set the cursor to any value. + /// + /// Should normally not be needed and is only in place as emergency measure. + #[pallet::call_index(0)] + pub fn force_set_cursor( + origin: OriginFor, + cursor: Option>, + ) -> DispatchResult { + ensure_root(origin)?; + + Cursor::::set(cursor); + + Ok(()) + } + + /// Clears the `Historic` set. + /// + /// `map_cursor` must be set to the last value that was returned by the + /// `HistoricCleared` event. The first time `None` can be used. `limit` must be chosen in a + /// way that will result in a sensible weight. + #[pallet::call_index(1)] + #[pallet::weight({0})] // FAIL-CI + pub fn clear_historic( + origin: OriginFor, + limit: Option, + map_cursor: Option>, + ) -> DispatchResult { + ensure_root(origin)?; + + let next = Historic::::clear(limit.unwrap_or_default(), map_cursor.as_deref()); + Self::deposit_event(Event::HistoricCleared { next_cursor: next.maybe_cursor }); + + Ok(()) + } + } +} + +impl Pallet { + /// Onboard all new Multi-Block-Migrations and start the process of executing them. + /// + /// Should only be called once all previous migrations completed. + fn onboard_new_mbms() -> Weight { + if let Some(cursor) = Cursor::::get() { + log::error!(target: LOG, "Ongoing migrations interrupted - chain stuck"); + + let maybe_index = cursor.as_active().map(|c| c.index); + Self::upgrade_failed(maybe_index); + return T::WeightInfo::on_runtime_upgrade() + } + + let migrations = T::Migrations::len(); + log::info!(target: LOG, "Onboarding {migrations} MBM migrations"); + if migrations > 0 { + Cursor::::set(Some( + ActiveCursor { + index: 0, + inner_cursor: None, + started_at: System::::block_number(), + } + .into(), + )); + Self::deposit_event(Event::UpgradeStarted { migrations }); + T::OnMigrationUpdate::started(); + } + + T::WeightInfo::on_runtime_upgrade() + } + + /// Tries to make progress on the Multi-Block-Migrations process. + fn progress_mbms(n: T::BlockNumber) -> Weight { + let mut meter = WeightMeter::from_limit(T::ServiceWeight::get()); + meter.defensive_saturating_accrue(T::WeightInfo::on_init_base()); + + let mut cursor = match Cursor::::get() { + None => { + log::trace!(target: LOG, "[Block {n:?}] Waiting for cursor to become `Some`."); + return meter.consumed + }, + Some(MigrationCursor::Active(cursor)) => { + log::info!(target: LOG, "Progressing MBM migration #{}", cursor.index); + cursor + }, + Some(MigrationCursor::Stuck) => { + log::error!(target: LOG, "Migration stuck. Governance intervention required."); + return meter.consumed + }, + }; + debug_assert!(::is_upgrading()); + + for i in 0.. { + match Self::exec_migration(&mut meter, cursor, i == 0) { + None => return meter.consumed, + Some(ControlFlow::Break(last_cursor)) => { + cursor = last_cursor; + break + }, + Some(ControlFlow::Continue(next_cursor)) => { + cursor = next_cursor; + }, + } + } + + Cursor::::set(Some(cursor.into())); + + meter.consumed + } + + /// Try to make progress on the current migration. + /// + /// Returns whether processing should continue or break for this block. The `meter` contains the + /// remaining weight that can be consumed. + fn exec_migration( + meter: &mut WeightMeter, + mut cursor: ActiveCursorOf, + is_first: bool, + ) -> Option, ActiveCursorOf>> { + let Some(id) = T::Migrations::nth_id(cursor.index) else { + Self::deposit_event(Event::UpgradeCompleted); + Cursor::::kill(); + T::OnMigrationUpdate::completed(); + return None; + }; + if Historic::::contains_key(&id) { + Self::deposit_event(Event::MigrationSkippedHistoric { index: cursor.index }); + cursor.advance(System::::block_number()); + return Some(ControlFlow::Continue(cursor)) + } + + let blocks = System::::block_number().saturating_sub(cursor.started_at); + match T::Migrations::nth_transactional_step(cursor.index, cursor.inner_cursor.clone(), meter) { + Ok(Some(next_cursor)) => { + Self::deposit_event(Event::MigrationAdvanced { index: cursor.index, blocks }); + cursor.inner_cursor = Some(next_cursor); + + // We only progress one step per block. + if migration.max_steps().map_or(false, |max| blocks > max.into()) { + Self::deposit_event(Event::MigrationFailed { index: cursor.index, blocks }); + Self::upgrade_failed(Some(cursor.index)); + None + } else { + // A migration cannot progress more than one step per block, we therefore break. + Some(ControlFlow::Break(cursor)) + } + }, + Ok(None) => { + Self::deposit_event(Event::MigrationCompleted { index: cursor.index, blocks }); + Historic::::insert(&migration.id(), ()); + cursor.advance(System::::block_number()); + return Some(ControlFlow::Continue(cursor)) + }, + Err(SteppedMigrationError::InsufficientWeight { required }) => { + if is_first || required.any_gt(meter.limit) { + Self::deposit_event(Event::MigrationFailed { index: cursor.index, blocks }); + Self::upgrade_failed(Some(cursor.index)); + None + } else { + // Hope that it gets better in the next block. + Some(ControlFlow::Break(cursor)) + } + }, + Err(SteppedMigrationError::InvalidCursor | SteppedMigrationError::Failed) => { + Self::deposit_event(Event::MigrationFailed { index: cursor.index, blocks }); + Self::upgrade_failed(Some(cursor.index)); + return None + }, + } + } + + /// Fail the current runtime upgrade. + fn upgrade_failed(migration: Option) { + use FailedUpgradeHandling::*; + Self::deposit_event(Event::UpgradeFailed); + + match T::OnMigrationUpdate::failed(migration) { + KeepStuck => Cursor::::set(Some(MigrationCursor::Stuck)), + ForceUnstuck => Cursor::::kill(), + } + } +} + +impl MultiStepMigrator for Pallet { + fn is_upgrading() -> bool { + Cursor::::exists() + } + + fn step() -> Weight { + Self::progress_mbms(System::::block_number()) + } +} diff --git a/frame/migrations/src/mock.rs b/frame/migrations/src/mock.rs new file mode 100644 index 0000000000000..7d9a4cdc203fb --- /dev/null +++ b/frame/migrations/src/mock.rs @@ -0,0 +1,204 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(test)] + +use crate::{mock_helpers::*, Event, Historic}; + +use core::cell::RefCell; +#[use_attr] +use frame_support::derive_impl; +use frame_support::{ + macro_magic::use_attr, + migrations::*, + traits::{OnFinalize, OnInitialize}, + weights::Weight, +}; +use frame_system::EventRecord; +use sp_core::{Get, H256}; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; + +// Configure a mock runtime to test the pallet. +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system, + Migrations: crate, + } +); + +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] +impl frame_system::Config for Test { + type BaseCallFilter = frame_support::traits::Everything; + type RuntimeOrigin = RuntimeOrigin; + type RuntimeCall = RuntimeCall; + type RuntimeEvent = RuntimeEvent; + type PalletInfo = PalletInfo; + type OnSetCode = (); +} + +frame_support::parameter_types! { + pub const ServiceWeight: Weight = Weight::MAX.div(10); +} + +thread_local! { + /// The configs for the migrations to run. + pub static MIGRATIONS: RefCell> = RefCell::new(vec![]); +} + +/// Dynamically set the migrations to run. +pub struct MigrationsStorage; +impl Get>>> + for MigrationsStorage +{ + fn get() -> Vec>> + { + MIGRATIONS.with(|m| { + m.borrow() + .clone() + .into_iter() + .map(|(k, v)| { + Box::new(MockedMigration(k, v)) + as Box< + dyn SteppedMigration< + Cursor = MockedCursor, + Identifier = MockedIdentifier, + >, + > + }) + .collect() + }) + } +} + +impl MigrationsStorage { + /// Set the migrations to run. + pub fn set(migrations: Vec<(MockedMigrationKind, u32)>) { + MIGRATIONS.with(|m| *m.borrow_mut() = migrations); + } +} + +impl crate::Config for Test { + type RuntimeEvent = RuntimeEvent; + type Migrations = MigrationsStorage; + type Cursor = MockedCursor; + type Identifier = MockedIdentifier; + type OnMigrationUpdate = MockedOnMigrationUpdate; + type ServiceWeight = ServiceWeight; + type WeightInfo = (); +} + +// Build genesis storage according to the mock runtime. +pub fn new_test_ext() -> sp_io::TestExternalities { + sp_tracing::try_init_simple(); + frame_system::GenesisConfig::default().build_storage::().unwrap().into() +} + +/// Run this closure in test externalities. +pub fn test_closure(f: impl FnOnce() -> R) -> R { + let mut ext = new_test_ext(); + ext.execute_with(f) +} + +pub fn run_to_block(n: u32) { + while System::block_number() < n { + if System::block_number() > 1 { + Migrations::on_finalize(System::block_number()); + System::on_finalize(System::block_number()); + } + log::debug!("Block {}", System::block_number()); + System::set_block_number(System::block_number() + 1); + System::on_initialize(System::block_number()); + Migrations::on_initialize(System::block_number()); + // Executive calls this: + ::step(); + } +} + +// Traits to make using events less insufferable: +pub trait IntoRecord { + fn into_record(self) -> EventRecord<::RuntimeEvent, H256>; +} + +impl IntoRecord for Event { + fn into_record(self) -> EventRecord<::RuntimeEvent, H256> { + let re: ::RuntimeEvent = self.into(); + EventRecord { phase: frame_system::Phase::Initialization, event: re, topics: vec![] } + } +} + +pub trait IntoRecords { + fn into_records(self) -> Vec::RuntimeEvent, H256>>; +} + +impl IntoRecords for Vec { + fn into_records(self) -> Vec::RuntimeEvent, H256>> { + self.into_iter().map(|e| e.into_record()).collect() + } +} + +pub fn assert_events(events: Vec) { + pretty_assertions::assert_eq!(events.into_records(), System::events()); + System::reset_events(); +} + +frame_support::parameter_types! { + /// The number of started upgrades. + pub static UpgradesStarted: u32 = 0; + /// The number of completed upgrades. + pub static UpgradesCompleted: u32 = 0; + /// The migrations that failed. + pub static UpgradesFailed: Vec> = vec![]; + /// Return value of `MockedOnMigrationUpdate::failed`. + pub static FailedUpgradeResponse: FailedUpgradeHandling = FailedUpgradeHandling::KeepStuck; +} + +pub struct MockedOnMigrationUpdate; +impl OnMigrationUpdate for MockedOnMigrationUpdate { + fn started() { + log::info!("OnMigrationUpdate started"); + UpgradesStarted::mutate(|v| *v += 1); + } + + fn completed() { + log::info!("OnMigrationUpdate completed"); + UpgradesCompleted::mutate(|v| *v += 1); + } + + fn failed(migration: Option) -> FailedUpgradeHandling { + UpgradesFailed::mutate(|v| v.push(migration)); + let res = FailedUpgradeResponse::get(); + log::error!("OnMigrationUpdate failed at: {migration:?}, handling as {res:?}"); + res + } +} + +/// Returns the number of `(started, completed, failed)` upgrades and resets their numbers. +pub fn upgrades_started_completed_failed() -> (u32, u32, u32) { + (UpgradesStarted::take(), UpgradesCompleted::take(), UpgradesFailed::take().len() as u32) +} + +pub fn historic() -> Vec { + let mut historic = Historic::::iter_keys().collect::>(); + historic.sort(); + historic +} diff --git a/frame/migrations/src/mock_helpers.rs b/frame/migrations/src/mock_helpers.rs new file mode 100644 index 0000000000000..70863e1d5fa5f --- /dev/null +++ b/frame/migrations/src/mock_helpers.rs @@ -0,0 +1,94 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Helpers for std and no-std testing. Can be re-used by other crates. + +use super::*; + +use codec::Encode; +use sp_core::ConstU32; +use sp_runtime::BoundedVec; + +/// An opaque cursor of a migration. +pub type MockedCursor = BoundedVec>; +/// An opaque identifier of a migration. +pub type MockedIdentifier = BoundedVec>; + +/// How a [`MockedMigration`] should behave. +#[derive(Debug, Clone, Copy, Encode)] +#[allow(dead_code)] +pub enum MockedMigrationKind { + /// Succeed after its number of steps elapsed. + SucceedAfter, + /// Fail after its number of steps elapsed. + FailAfter, + /// Never terminate. + TimeoutAfter, + /// Cause an [`InsufficientWeight`] error after its number of steps elapsed. + HightWeightAfter(Weight), +} +use MockedMigrationKind::*; // C style + +/// A migration that does something after a certain number of steps. +pub struct MockedMigration(pub MockedMigrationKind, pub u32); + +impl SteppedMigration for MockedMigration { + type Cursor = MockedCursor; + type Identifier = MockedIdentifier; + + fn id(&self) -> Self::Identifier { + mocked_id(self.0, self.1) + } + + fn max_steps(&self) -> Option { + matches!(self.0, TimeoutAfter).then(|| self.1) + } + + fn step( + cursor: &Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + let mut count: u32 = + cursor.as_ref().and_then(|c| Decode::decode(&mut &c[..]).ok()).unwrap_or(0); + log::debug!("MockedMigration: Step {}", count); + if count != self.1 || matches!(self.0, TimeoutAfter) { + count += 1; + return Ok(Some(count.encode().try_into().unwrap())) + } + + match self.0 { + SucceedAfter => { + log::debug!("MockedMigration: Succeeded after {} steps", count); + Ok(None) + }, + HightWeightAfter(required) => { + log::debug!("MockedMigration: Not enough weight after {} steps", count); + Err(SteppedMigrationError::InsufficientWeight { required }) + }, + FailAfter => { + log::debug!("MockedMigration: Failed after {} steps", count); + Err(SteppedMigrationError::Failed) + }, + TimeoutAfter => unreachable!(), + } + } +} + +/// Calculate the identifier of a mocked migration. +pub fn mocked_id(kind: MockedMigrationKind, steps: u32) -> MockedIdentifier { + (b"MockedMigration", kind, steps).encode().try_into().unwrap() +} diff --git a/frame/migrations/src/tests.rs b/frame/migrations/src/tests.rs new file mode 100644 index 0000000000000..d2a6e5202d600 --- /dev/null +++ b/frame/migrations/src/tests.rs @@ -0,0 +1,370 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(test)] + +use crate::{ + mock::{Test as T, *}, + mock_helpers::{MockedMigrationKind::*, *}, + Cursor, Event, FailedUpgradeHandling, MigrationCursor, +}; +use frame_support::{pallet_prelude::Weight, traits::OnRuntimeUpgrade}; + +#[test] +#[docify::export] +fn simple_works() { + use Event::*; + test_closure(|| { + // Add three migrations, each taking one block longer than the last. + MigrationsStorage::set(vec![(SucceedAfter, 0), (SucceedAfter, 1), (SucceedAfter, 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Check that the executed migrations are recorded in `Historical`. + assert_eq!( + historic(), + vec![ + mocked_id(SucceedAfter, 0), + mocked_id(SucceedAfter, 1), + mocked_id(SucceedAfter, 2), + ] + ); + // Check that we got all events. + assert_events(vec![ + UpgradeStarted { migrations: 3 }, + MigrationCompleted { index: 0, blocks: 1 }, + MigrationAdvanced { index: 1, blocks: 0 }, + MigrationCompleted { index: 1, blocks: 1 }, + MigrationAdvanced { index: 2, blocks: 0 }, + MigrationAdvanced { index: 2, blocks: 1 }, + MigrationCompleted { index: 2, blocks: 2 }, + UpgradeCompleted, + ]); + }); +} + +#[test] +fn basic_works() { + test_closure(|| { + MigrationsStorage::set(vec![(SucceedAfter, 0), (SucceedAfter, 1), (SucceedAfter, 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Check that the executed migrations are recorded into `Historical`. + assert_eq!( + historic(), + vec![ + mocked_id(SucceedAfter, 0), + mocked_id(SucceedAfter, 1), + mocked_id(SucceedAfter, 2), + ] + ); + // Check that we got all events. + assert_events(vec![ + Event::UpgradeStarted { migrations: 3 }, + Event::MigrationCompleted { index: 0, blocks: 1 }, + Event::MigrationAdvanced { index: 1, blocks: 0 }, + Event::MigrationCompleted { index: 1, blocks: 1 }, + Event::MigrationAdvanced { index: 2, blocks: 0 }, + Event::MigrationAdvanced { index: 2, blocks: 1 }, + Event::MigrationCompleted { index: 2, blocks: 2 }, + Event::UpgradeCompleted, + ]); + }); +} + +#[test] +fn failing_migration_keep_stuck_the_chain() { + test_closure(|| { + FailedUpgradeResponse::set(FailedUpgradeHandling::KeepStuck); + MigrationsStorage::set(vec![(FailAfter, 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Failed migrations are not recorded in `Historical`. + assert!(historic().is_empty()); + // Check that we got all events. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, blocks: 1 }, + Event::MigrationAdvanced { index: 0, blocks: 2 }, + Event::MigrationFailed { index: 0, blocks: 3 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(UpgradesStarted::take(), 1); + assert_eq!(UpgradesCompleted::take(), 0); + assert_eq!(UpgradesFailed::take(), vec![Some(0)]); + + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck), "Must stuck the chain"); + }); +} + +#[test] +fn failing_migration_force_unstuck_the_chain() { + test_closure(|| { + FailedUpgradeResponse::set(FailedUpgradeHandling::ForceUnstuck); + MigrationsStorage::set(vec![(FailAfter, 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Failed migrations are not recorded in `Historical`. + assert!(historic().is_empty()); + // Check that we got all events. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, blocks: 1 }, + Event::MigrationAdvanced { index: 0, blocks: 2 }, + Event::MigrationFailed { index: 0, blocks: 3 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(UpgradesStarted::take(), 1); + assert_eq!(UpgradesCompleted::take(), 0); + assert_eq!(UpgradesFailed::take(), vec![Some(0)]); + + assert!(Cursor::::get().is_none(), "Must unstuck the chain"); + }); +} + +/// A migration that reports not getting enough weight errors if it is the first one to run in that +/// block. +#[test] +fn high_weight_migration_singular_fails() { + test_closure(|| { + MigrationsStorage::set(vec![(HightWeightAfter(Weight::zero()), 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Failed migrations are not recorded in `Historical`. + assert!(historic().is_empty()); + // Check that we got all events. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, blocks: 1 }, + Event::MigrationAdvanced { index: 0, blocks: 2 }, + Event::MigrationFailed { index: 0, blocks: 3 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + }); +} + +/// A migration that reports of not getting enough weight is retried once, if it is not the first +/// one to run in a block. +#[test] +fn high_weight_migration_retries_once() { + test_closure(|| { + MigrationsStorage::set(vec![(SucceedAfter, 0), (HightWeightAfter(Weight::zero()), 0)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + assert_eq!(historic(), vec![mocked_id(SucceedAfter, 0)]); + // Check that we got all events. + assert_events::>(vec![ + Event::UpgradeStarted { migrations: 2 }, + Event::MigrationCompleted { index: 0, blocks: 1 }, + // `blocks=1` means that it was retried once. + Event::MigrationFailed { index: 1, blocks: 1 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + }); +} + +/// If a migration uses more weight than the limit, then it will not retry but fail even when it is +/// not the first one in the block. +// Note: Same as `high_weight_migration_retries_once` but with different required weight for the +// migration. +#[test] +fn high_weight_migration_permanently_overweight_fails() { + test_closure(|| { + MigrationsStorage::set(vec![(SucceedAfter, 0), (HightWeightAfter(Weight::MAX), 0)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + assert_eq!(historic(), vec![mocked_id(SucceedAfter, 0)]); + // Check that we got all events. + assert_events::>(vec![ + Event::UpgradeStarted { migrations: 2 }, + Event::MigrationCompleted { index: 0, blocks: 1 }, + // `blocks=0` means that it was not retried. + Event::MigrationFailed { index: 1, blocks: 0 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + }); +} + +#[test] +fn historic_skipping_works() { + test_closure(|| { + MigrationsStorage::set(vec![ + (SucceedAfter, 0), + (SucceedAfter, 0), // duplicate + (SucceedAfter, 1), + (SucceedAfter, 2), + (SucceedAfter, 1), // duplicate + ]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Just three historical ones, since two were added twice. + assert_eq!( + historic(), + vec![ + mocked_id(SucceedAfter, 0), + mocked_id(SucceedAfter, 1), + mocked_id(SucceedAfter, 2), + ] + ); + // Events received. + assert_events(vec![ + Event::UpgradeStarted { migrations: 5 }, + Event::MigrationCompleted { index: 0, blocks: 1 }, + Event::MigrationSkippedHistoric { index: 1 }, + Event::MigrationAdvanced { index: 2, blocks: 0 }, + Event::MigrationCompleted { index: 2, blocks: 1 }, + Event::MigrationAdvanced { index: 3, blocks: 0 }, + Event::MigrationAdvanced { index: 3, blocks: 1 }, + Event::MigrationCompleted { index: 3, blocks: 2 }, + Event::MigrationSkippedHistoric { index: 4 }, + Event::UpgradeCompleted, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 1, 0)); + + // Now go for another upgrade; just to make sure that it wont execute again. + System::reset_events(); + Migrations::on_runtime_upgrade(); + run_to_block(20); + + // Same historical ones as before. + assert_eq!( + historic(), + vec![ + mocked_id(SucceedAfter, 0), + mocked_id(SucceedAfter, 1), + mocked_id(SucceedAfter, 2), + ] + ); + + // Everything got skipped. + assert_events(vec![ + Event::UpgradeStarted { migrations: 5 }, + Event::MigrationSkippedHistoric { index: 0 }, + Event::MigrationSkippedHistoric { index: 1 }, + Event::MigrationSkippedHistoric { index: 2 }, + Event::MigrationSkippedHistoric { index: 3 }, + Event::MigrationSkippedHistoric { index: 4 }, + Event::UpgradeCompleted, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 1, 0)); + }); +} + +/// When another upgrade happens while a migration is still running, it should set the cursor to +/// stuck. +#[test] +fn upgrade_fails_when_migration_active() { + test_closure(|| { + MigrationsStorage::set(vec![(SucceedAfter, 10)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(3); + + //assert_eq!( // TODO + // historic(), + // vec![mocked_id(SucceedAfter, 0)] + //); + // Events received. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, blocks: 1 }, + Event::MigrationAdvanced { index: 0, blocks: 2 }, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 0, 0)); + + // Upgrade again. + Migrations::on_runtime_upgrade(); + // -- Defensive path -- + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + assert_events(vec![Event::UpgradeFailed]); + assert_eq!(upgrades_started_completed_failed(), (0, 0, 1)); + }); +} + +#[test] +fn migration_timeout_errors() { + test_closure(|| { + MigrationsStorage::set(vec![(TimeoutAfter, 3)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(5); + + // Times out after taking more than 3 steps. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, blocks: 1 }, + Event::MigrationAdvanced { index: 0, blocks: 2 }, + Event::MigrationAdvanced { index: 0, blocks: 3 }, + Event::MigrationAdvanced { index: 0, blocks: 4 }, + Event::MigrationFailed { index: 0, blocks: 4 }, + Event::UpgradeFailed, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + + // Failed migrations are not black-listed. + assert!(historic().is_empty()); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + + Migrations::on_runtime_upgrade(); + run_to_block(6); + + assert_events(vec![Event::UpgradeFailed]); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + assert_eq!(upgrades_started_completed_failed(), (0, 0, 1)); + }); +} diff --git a/frame/migrations/src/weights.rs b/frame/migrations/src/weights.rs new file mode 100644 index 0000000000000..3ecd45f792906 --- /dev/null +++ b/frame/migrations/src/weights.rs @@ -0,0 +1,139 @@ + +//! Autogenerated weights for pallet_migrations +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2023-06-02, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `i9`, CPU: `13th Gen Intel(R) Core(TM) i9-13900K` +//! EXECUTION: None, WASM-EXECUTION: Compiled, CHAIN: None, DB CACHE: 1024 + +// Executed Command: +// ./target/release/substrate +// benchmark +// pallet +// --pallet +// pallet_migrations +// --extrinsic +// +// --output +// frame/migrations/src/weights.rs +// --template +// .maintain/frame-weight-template.hbs +// --steps +// 50 +// --repeat +// 20 + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use core::marker::PhantomData; + +/// Weight functions needed for pallet_migrations. +pub trait WeightInfo { + fn on_runtime_upgrade() -> Weight; + fn on_init_base() -> Weight; + fn on_init_fast_path() -> Weight; + fn on_init_loop_base() -> Weight; + fn force_set_cursor() -> Weight; +} + +/// Weights for pallet_migrations using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + fn on_init_fast_path() -> Weight { + Weight::zero() + } + /// Storage: MultiBlockMigrations Cursor (r:1 w:0) + /// Proof: MultiBlockMigrations Cursor (max_values: Some(1), max_size: Some(10), added: 505, mode: MaxEncodedLen) + fn on_runtime_upgrade() -> Weight { + // Proof Size summary in bytes: + // Measured: `109` + // Estimated: `1495` + // Minimum execution time: 1_097_000 picoseconds. + Weight::from_parts(1_183_000, 1495) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + /// Storage: MultiBlockMigrations Cursor (r:1 w:1) + /// Proof: MultiBlockMigrations Cursor (max_values: Some(1), max_size: Some(10), added: 505, mode: MaxEncodedLen) + fn on_init_base() -> Weight { + // Proof Size summary in bytes: + // Measured: `143` + // Estimated: `1495` + // Minimum execution time: 3_220_000 picoseconds. + Weight::from_parts(3_463_000, 1495) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: MultiBlockMigrations Cursor (r:1 w:0) + /// Proof: MultiBlockMigrations Cursor (max_values: Some(1), max_size: Some(10), added: 505, mode: MaxEncodedLen) + fn on_init_loop_base() -> Weight { + // Proof Size summary in bytes: + // Measured: `109` + // Estimated: `1495` + // Minimum execution time: 1_307_000 picoseconds. + Weight::from_parts(1_368_000, 1495) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + /// Storage: MultiBlockMigrations Cursor (r:0 w:1) + /// Proof: MultiBlockMigrations Cursor (max_values: Some(1), max_size: Some(10), added: 505, mode: MaxEncodedLen) + fn force_set_cursor() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 866_000 picoseconds. + Weight::from_parts(946_000, 0) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + /// Storage: MultiBlockMigrations Cursor (r:1 w:0) + /// Proof: MultiBlockMigrations Cursor (max_values: Some(1), max_size: Some(10), added: 505, mode: MaxEncodedLen) + fn on_runtime_upgrade() -> Weight { + // Proof Size summary in bytes: + // Measured: `109` + // Estimated: `1495` + // Minimum execution time: 1_097_000 picoseconds. + Weight::from_parts(1_183_000, 1495) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + } + /// Storage: MultiBlockMigrations Cursor (r:1 w:1) + /// Proof: MultiBlockMigrations Cursor (max_values: Some(1), max_size: Some(10), added: 505, mode: MaxEncodedLen) + fn on_init_base() -> Weight { + // Proof Size summary in bytes: + // Measured: `143` + // Estimated: `1495` + // Minimum execution time: 3_220_000 picoseconds. + Weight::from_parts(3_463_000, 1495) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: MultiBlockMigrations Cursor (r:1 w:0) + /// Proof: MultiBlockMigrations Cursor (max_values: Some(1), max_size: Some(10), added: 505, mode: MaxEncodedLen) + fn on_init_loop_base() -> Weight { + // Proof Size summary in bytes: + // Measured: `109` + // Estimated: `1495` + // Minimum execution time: 1_307_000 picoseconds. + Weight::from_parts(1_368_000, 1495) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + } + /// Storage: MultiBlockMigrations Cursor (r:0 w:1) + /// Proof: MultiBlockMigrations Cursor (max_values: Some(1), max_size: Some(10), added: 505, mode: MaxEncodedLen) + fn force_set_cursor() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 866_000 picoseconds. + Weight::from_parts(946_000, 0) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + fn on_init_fast_path() -> Weight { + Weight::zero() + } +} diff --git a/frame/support/procedural/src/construct_runtime/expand/inherent.rs b/frame/support/procedural/src/construct_runtime/expand/inherent.rs index 2f1cf75ab7ce9..9f089893e485b 100644 --- a/frame/support/procedural/src/construct_runtime/expand/inherent.rs +++ b/frame/support/procedural/src/construct_runtime/expand/inherent.rs @@ -178,12 +178,12 @@ pub fn expand_outer_inherent( } impl #scrate::traits::EnsureInherentsAreFirst<#block> for #runtime { - fn ensure_inherents_are_first(block: &#block) -> Result<(), u32> { + fn ensure_inherents_are_first(block: &#block) -> Result { use #scrate::inherent::ProvideInherent; use #scrate::traits::{IsSubType, ExtrinsicCall}; use #scrate::sp_runtime::traits::Block as _; - let mut first_signed_observed = false; + let mut first_signed_index = None; for (i, xt) in block.extrinsics().iter().enumerate() { let is_signed = #scrate::sp_runtime::traits::Extrinsic::is_signed(xt) @@ -208,16 +208,16 @@ pub fn expand_outer_inherent( is_inherent }; - if !is_inherent { - first_signed_observed = true; - } - - if first_signed_observed && is_inherent { - return Err(i as u32) + if is_inherent { + if first_signed_index.is_some() { + return Err(i as u32) + } + } else if first_signed_index.is_none() { + first_signed_index = Some(i as u32); } } - Ok(()) + Ok(first_signed_index.unwrap_or(0)) } } } diff --git a/frame/support/src/migrations.rs b/frame/support/src/migrations.rs index 8bda2662a237e..ad7c311e7914f 100644 --- a/frame/support/src/migrations.rs +++ b/frame/support/src/migrations.rs @@ -18,9 +18,11 @@ #[cfg(feature = "try-runtime")] use crate::storage::unhashed::contains_prefixed_key; use crate::{ + storage::transactional::with_transaction_opaque_err, traits::{GetStorageVersion, NoStorageVersionSet, PalletInfoAccess, StorageVersion}, - weights::{RuntimeDbWeight, Weight}, + weights::{RuntimeDbWeight, Weight, WeightMeter}, }; +use codec::{Decode, Encode, MaxEncodedLen}; use impl_trait_for_tuples::impl_for_tuples; use sp_core::Get; use sp_io::{hashing::twox_128, storage::clear_prefix, KillStorageResult}; @@ -209,3 +211,315 @@ impl, DbWeight: Get> frame_support::traits Ok(()) } } + +/// A migration that can proceed in multiple steps. +pub trait SteppedMigration { + /// The cursor type that stores the progress (aka. state) of this migration. + type Cursor: codec::FullCodec + codec::MaxEncodedLen; + + /// The unique identifier type of this migration. + type Identifier: codec::FullCodec + codec::MaxEncodedLen; + + /// The unique identifier of this migration. + /// + /// If two migrations have the same identifier, then they are assumed to be identical. + fn id() -> Self::Identifier; + + /// The maximum number of steps that this migration can take at most. + /// + /// This can be used to enforce progress and prevent migrations to be stuck forever. A migration + /// that exceeds its max steps is treated as failed. `None` means that there is no limit. + fn max_steps() -> Option { + None + } + + /// Try to migrate as much as possible with the given weight. + /// + /// **ANY STORAGE CHANGES MUST BE ROLLED-BACK BY THE CALLER UPON ERROR.** This is necessary + /// since the caller cannot return a cursor in the error case. `Self::transactional_step` is + /// provided as convenience for a caller. A cursor of `None` implies that the migration is at + /// its end. TODO: Think about iterator `fuse` requirement. + fn step( + cursor: Option, + meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError>; + + /// Same as [`Self::step`], but rolls back pending changes in the error case. + fn transactional_step( + mut cursor: Option, + meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + with_transaction_opaque_err(move || match Self::step(cursor, meter) { + Ok(new_cursor) => { + cursor = new_cursor; + sp_api::TransactionOutcome::Commit(Ok(cursor)) + }, + Err(err) => sp_api::TransactionOutcome::Rollback(Err(err)), + }) + .map_err(|()| SteppedMigrationError::Failed)? + } +} + +#[derive(Debug, Encode, Decode, MaxEncodedLen, scale_info::TypeInfo)] +pub enum SteppedMigrationError { + // Transient errors: + /// The remaining weight is not enough to do anything. + /// + /// Can be resolved by calling with at least `required` weight. Note that calling it with + /// exactly `required` weight could cause it to not make any progress. + InsufficientWeight { required: Weight }, + // Permanent errors: + /// The migration cannot decode its cursor and therefore not proceed. + /// + /// This should not happen unless (1) the migration itself returned an invalid cursor in a + /// previous iteration, (2) the storage got corrupted or (3) there is a bug in the caller's + /// code. + InvalidCursor, + /// The migration encountered a permanent error and cannot continue. + Failed, +} + +/// Notification handler for status updates regarding runtime upgrades. +pub trait OnMigrationUpdate { + /// Notifies of the start of a runtime upgrade. + fn started() {} + + /// Notifies of the completion of a runtime upgrade. + fn completed() {} + + /// Infallibly handle a failed runtime upgrade. + /// + /// Gets passed in the optional index of the migration that caused the failure. + fn failed(migration: Option) -> FailedUpgradeHandling; +} + +/// How to proceed after a runtime upgrade failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FailedUpgradeHandling { + /// Resume extrinsic processing of the chain. This will not resume the upgrade. + /// + /// This should be supplemented with additional measures to ensure that the broken chain state + /// does not get further messed up by user extrinsics. + ForceUnstuck, + /// Do nothing and keep blocking extrinsics. + KeepStuck, +} + +impl OnMigrationUpdate for () { + fn failed(_migration: Option) -> FailedUpgradeHandling { + FailedUpgradeHandling::KeepStuck + } +} + +/// Something that can do multi step migrations. +pub trait MultiStepMigrator { + /// Hint for whether [`step`] should be called. + fn is_upgrading() -> bool; + /// Do the next step in the MBM process. + /// + /// Must gracefully handle the case that it is currently not upgrading. + fn step() -> Weight; +} + +impl MultiStepMigrator for () { + fn is_upgrading() -> bool { + false + } + + fn step() -> Weight { + Weight::zero() + } +} + +/// Multiple [`SteppedMigration`]. +pub trait SteppedMigrations { + fn len() -> u32; + + fn nth_id(n: u32) -> Option>; + + fn nth_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>>; + + fn nth_transactional_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>>; +} + +impl SteppedMigrations for T { + fn len() -> u32 { + 1 + } + + fn nth_id(_: u32) -> Option> { + Some(T::id().encode()) + } + + fn nth_step( + _: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + Some( + T::step(cursor.map(|c| Decode::decode(&mut &c[..]).unwrap()), meter) + .map(|v| v.map(|v| v.encode())), + ) + } + + fn nth_transactional_step( + _: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + Some( + T::transactional_step(cursor.map(|c| Decode::decode(&mut &c[..]).unwrap()), meter) + .map(|v| v.map(|v| v.encode())), + ) + } +} + +#[impl_trait_for_tuples::impl_for_tuples(1, 30)] +impl SteppedMigrations for Tuple { + fn len() -> u32 { + for_tuples!( #( Tuple::len() )+* ) + } + + fn nth_id(n: u32) -> Option> { + let mut i = 0; + + for_tuples!( #( + if (i + Tuple::len()) > n { + return Tuple::nth_id(n - i) + } + + i += Tuple::len(); + )* ); + + None + } + + fn nth_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + let mut i = 0; + + for_tuples!( #( + if (i + Tuple::len()) > n { + return Tuple::nth_step(n - i, cursor, meter) + } + + i += Tuple::len(); + )* ); + + None + } + + fn nth_transactional_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + let mut i = 0; + + for_tuples! ( #( + if (i + Tuple::len()) > n { + return Tuple::nth_transactional_step(n - i, cursor, meter) + } + + i += Tuple::len(); + )* ); + + None + } +} + +#[derive(Decode, Encode, MaxEncodedLen, Eq, PartialEq)] +pub enum Either { + Left(L), + Right(R), +} + +pub struct M0; +impl SteppedMigration for M0 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 0 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("M0"); + Ok(None) + } +} + +pub struct M1; +impl SteppedMigration for M1 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 1 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("M1"); + Ok(None) + } +} + +pub struct M2; +impl SteppedMigration for M2 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 2 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("M2"); + Ok(None) + } +} + +#[test] +fn templates_work() { + sp_tracing::init_for_tests(); + // Three migrations combined to execute in order: + type Triple = (M0, (M1, M2)); + assert_eq!(::len(), 3); + // Six migrations, just concatenating the ones from before: + type Hextuple = (Triple, Triple); + assert_eq!(::len(), 6); + + // Check the IDs. The index specific functions all return an Option, + // to account for the out-of-range case. + assert_eq!(::nth_id(0), Some(0u8.encode())); + assert_eq!(::nth_id(1), Some(1u8.encode())); + assert_eq!(::nth_id(2), Some(2u8.encode())); + + for n in 0..3 { + ::nth_step( + n, + Default::default(), + &mut WeightMeter::max_limit(), + ); + } +} diff --git a/frame/support/src/storage/transactional.rs b/frame/support/src/storage/transactional.rs index d42e1809e9129..0671db4a3a86b 100644 --- a/frame/support/src/storage/transactional.rs +++ b/frame/support/src/storage/transactional.rs @@ -127,6 +127,22 @@ where } } +/// Same as [`with_transaction`] but casts any internal error to `()`. +/// +/// This rids `E` of the `From` bound that is required by `with_transaction`. +pub fn with_transaction_opaque_err(f: F) -> Result, ()> +where + F: FnOnce() -> TransactionOutcome>, +{ + with_transaction(move || -> TransactionOutcome, DispatchError>> { + match f() { + TransactionOutcome::Commit(res) => TransactionOutcome::Commit(Ok(res)), + TransactionOutcome::Rollback(res) => TransactionOutcome::Rollback(Ok(res)), + } + }) + .map_err(|_| ()) +} + /// Same as [`with_transaction`] but without a limit check on nested transactional layers. /// /// This is mostly for backwards compatibility before there was a transactional layer limit. diff --git a/frame/support/src/traits/misc.rs b/frame/support/src/traits/misc.rs index a6f8c46d63951..e3a51aa2a678d 100644 --- a/frame/support/src/traits/misc.rs +++ b/frame/support/src/traits/misc.rs @@ -880,8 +880,10 @@ pub trait GetBacking { pub trait EnsureInherentsAreFirst { /// Ensure the position of inherent is correct, i.e. they are before non-inherents. /// - /// On error return the index of the inherent with invalid position (counting from 0). - fn ensure_inherents_are_first(block: &Block) -> Result<(), u32>; + /// On error return the index of the inherent with invalid position (counting from 0). On + /// success it returns the index of the last inherent. `0` therefore means that there are no + /// inherents. + fn ensure_inherents_are_first(block: &Block) -> Result; } /// An extrinsic on which we can get access to call. diff --git a/frame/system/benchmarking/src/lib.rs b/frame/system/benchmarking/src/lib.rs index 1cd7b1bac6bd5..ccd3bacf62ec6 100644 --- a/frame/system/benchmarking/src/lib.rs +++ b/frame/system/benchmarking/src/lib.rs @@ -18,6 +18,7 @@ // Benchmarks for Utility Pallet #![cfg_attr(not(feature = "std"), no_std)] +#![cfg(feature = "runtime-benchmarks")] use codec::Encode; use frame_benchmarking::v1::{benchmarks, whitelisted_caller}; diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index cbda3d55cc68c..d095b6700c436 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -444,6 +444,9 @@ pub mod pallet { } /// Set the new runtime code without doing any checks of the given `code`. + /// + /// Note that runtime upgrades will not run if this is called with a not-increasing spec + /// version! #[pallet::call_index(3)] #[pallet::weight((T::SystemWeightInfo::set_code(), DispatchClass::Operational))] pub fn set_code_without_checks( diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index 1a286a927e6e8..20b292ae29ed0 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -287,7 +287,7 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// # unimplemented!() /// # } /// # fn execute_block(_block: Block) {} -/// # fn initialize_block(_header: &::Header) {} +/// # fn initialize_block(_header: &::Header) { } /// # } /// /// impl self::Balance for Runtime { @@ -715,7 +715,7 @@ pub fn deserialize_runtime_api_info(bytes: [u8; RUNTIME_API_INFO_SIZE]) -> ([u8; decl_runtime_apis! { /// The `Core` runtime api that every Substrate runtime needs to implement. #[core_trait] - #[api_version(4)] + #[api_version(5)] pub trait Core { /// Returns the version of the runtime. fn version() -> RuntimeVersion; diff --git a/primitives/block-builder/src/lib.rs b/primitives/block-builder/src/lib.rs index 29e04857f463e..8681953778330 100644 --- a/primitives/block-builder/src/lib.rs +++ b/primitives/block-builder/src/lib.rs @@ -20,11 +20,11 @@ #![cfg_attr(not(feature = "std"), no_std)] use sp_inherents::{CheckInherentsResult, InherentData}; -use sp_runtime::{traits::Block as BlockT, ApplyExtrinsicResult}; +use sp_runtime::{traits::Block as BlockT, ApplyExtrinsicResult, BlockAfterInherentsMode}; sp_api::decl_runtime_apis! { /// The `BlockBuilder` api trait that provides the required functionality for building a block. - #[api_version(6)] + #[api_version(7)] pub trait BlockBuilder { /// Apply the given extrinsic. /// @@ -48,5 +48,8 @@ sp_api::decl_runtime_apis! { /// Check that the inherents are valid. The inherent data will vary from chain to chain. fn check_inherents(block: Block, data: InherentData) -> CheckInherentsResult; + + /// Called after inherents are done but before extrinsic processing. + fn after_inherents() -> BlockAfterInherentsMode; } } diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 56e4efcad2c05..e96b4172689e3 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -557,6 +557,8 @@ pub enum DispatchError { Unavailable, /// Root origin is not allowed. RootNotAllowed, + /// The runtime is suspended and marks all transactions as failed. + Suspended, } /// Result of a `Dispatchable` which contains the `DispatchResult` and additional information about @@ -686,6 +688,7 @@ impl From for &'static str { Corruption => "State corrupt", Unavailable => "Resource unavailable", RootNotAllowed => "Root not allowed", + Suspended => "Extrinsics suspended", } } } @@ -733,6 +736,7 @@ impl traits::Printable for DispatchError { Corruption => "State corrupt".print(), Unavailable => "Resource unavailable".print(), RootNotAllowed => "Root not allowed".print(), + Suspended => "Extrinsics suspended".print(), } } } @@ -934,6 +938,15 @@ impl TransactionOutcome { } } +/// The mode of a block after inherents were applied. +#[derive(Debug, PartialEq, Eq, Clone, Copy, Encode, Decode, TypeInfo)] +pub enum BlockAfterInherentsMode { + /// No extrinsics should be pushed to the block. + ExtrinsicsForbidden, + /// Can push extrinsics to the block. + ExtrinsicsAllowed, +} + #[cfg(test)] mod tests { use crate::traits::BlakeTwo256; diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 8d77439f16455..bacc4a2d4aa67 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -480,7 +480,7 @@ impl_runtime_apis! { fn initialize_block(header: &::Header) { log::trace!(target: LOG_TARGET, "initialize_block: {header:#?}"); - Executive::initialize_block(header); + Executive::initialize_block(header) } } @@ -526,6 +526,10 @@ impl_runtime_apis! { fn check_inherents(_block: Block, _data: InherentData) -> CheckInherentsResult { CheckInherentsResult::new() } + + fn after_inherents() -> sp_runtime::BlockAfterInherentsMode { + Executive::after_inherents() + } } impl frame_system_rpc_runtime_api::AccountNonceApi for Runtime {