Allow import withouth state verification (#4031)

* Allow import without state verification

* Explicit None

Co-Authored-By: Robert Habermeier <rphmeier@gmail.com>

core/client/db/src/lib.rs

@@ -457,6 +457,7 @@ pub struct BlockImportOperation<Block: BlockT, H: Hasher> {
 	aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
 	finalized_blocks: Vec<(BlockId<Block>, Option<Justification>)>,
 	set_head: Option<BlockId<Block>>,
+	commit_state: bool,
 }
 
 impl<Block: BlockT, H: Hasher> BlockImportOperation<Block, H> {
@@ -531,6 +532,7 @@ impl<Block> client::backend::BlockImportOperation<Block, Blake2Hasher>
 		);
 
 		self.db_updates = transaction;
+		self.commit_state = true;
 		Ok(root)
 	}
 
@@ -783,6 +785,7 @@ pub struct Backend<Block: BlockT> {
 	canonicalization_delay: u64,
 	shared_cache: SharedCache<Block, Blake2Hasher>,
 	import_lock: Mutex<()>,
+	is_archive: bool,
 }
 
 impl<Block: BlockT<Hash=H256>> Backend<Block> {
@@ -843,6 +846,7 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 				config.state_cache_child_ratio.unwrap_or(DEFAULT_CHILD_RATIO),
 			),
 			import_lock: Default::default(),
+			is_archive: is_archive_pruning,
 		})
 	}
 
@@ -894,6 +898,12 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 		inmem
 	}
 
+	/// Returns total numbet of blocks (headers) in the block DB.
+	#[cfg(feature = "test-helpers")]
+	pub fn blocks_count(&self) -> u64 {
+		self.blockchain.db.iter(columns::HEADER).count() as u64
+	}
+
 	/// Read (from storage or cache) changes trie config.
 	///
 	/// Currently changes tries configuration is set up once (at genesis) and could not
@@ -1115,7 +1125,7 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 			);
 
 			transaction.put(columns::HEADER, &lookup_key, &pending_block.header.encode());
-			if let Some(body) = pending_block.body {
+			if let Some(body) = &pending_block.body {
 				transaction.put(columns::BODY, &lookup_key, &body.encode());
 			}
 			if let Some(justification) = pending_block.justification {
@@ -1127,21 +1137,26 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 				transaction.put(columns::META, meta_keys::GENESIS_HASH, hash.as_ref());
 			}
 
-			let mut changeset: state_db::ChangeSet<Vec<u8>> = state_db::ChangeSet::default();
-			for (key, (val, rc)) in operation.db_updates.drain() {
-				if rc > 0 {
-					changeset.inserted.push((key, val.to_vec()));
-				} else if rc < 0 {
-					changeset.deleted.push(key);
+			let finalized = if operation.commit_state {
+				let mut changeset: state_db::ChangeSet<Vec<u8>> = state_db::ChangeSet::default();
+				for (key, (val, rc)) in operation.db_updates.drain() {
+					if rc > 0 {
+						changeset.inserted.push((key, val.to_vec()));
+					} else if rc < 0 {
+						changeset.deleted.push(key);
+					}
 				}
-			}
-			let number_u64 = number.saturated_into::<u64>();
-			let commit = self.storage.state_db.insert_block(&hash, number_u64, &pending_block.header.parent_hash(), changeset)
-				.map_err(|e: state_db::Error<io::Error>| client::error::Error::from(format!("State database error: {:?}", e)))?;
-			apply_state_commit(&mut transaction, commit);
-
-			// Check if need to finalize. Genesis is always finalized instantly.
-			let finalized = number_u64 == 0 || pending_block.leaf_state.is_final();
+				let number_u64 = number.saturated_into::<u64>();
+				let commit = self.storage.state_db.insert_block(&hash, number_u64, &pending_block.header.parent_hash(), changeset)
+					.map_err(|e: state_db::Error<io::Error>| client::error::Error::from(format!("State database error: {:?}", e)))?;
+				apply_state_commit(&mut transaction, commit);
+
+				// Check if need to finalize. Genesis is always finalized instantly.
+				let finalized = number_u64 == 0 || pending_block.leaf_state.is_final();
+				finalized
+			} else {
+				false
+			};
 
 			let header = &pending_block.header;
 			let is_best = pending_block.leaf_state.is_best();
@@ -1347,6 +1362,7 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 			aux_ops: Vec::new(),
 			finalized_blocks: Vec::new(),
 			set_head: None,
+			commit_state: false,
 		})
 	}
 
@@ -1356,6 +1372,7 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 		block: BlockId<Block>,
 	) -> ClientResult<()> {
 		operation.old_state = self.state_at(block)?;
+		operation.commit_state = true;
 		Ok(())
 	}
 
@@ -1478,6 +1495,9 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 		match self.blockchain.header(block) {
 			Ok(Some(ref hdr)) => {
 				let hash = hdr.hash();
+				if !self.have_state_at(&hash, *hdr.number()) {
+					return Err(client::error::Error::UnknownBlock(format!("State already discarded for {:?}", block)))
+				}
 				if let Ok(()) = self.storage.state_db.pin(&hash) {
 					let root = H256::from_slice(hdr.state_root().as_ref());
 					let db_state = DbState::new(self.storage.clone(), root);
@@ -1493,7 +1513,16 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 	}
 
 	fn have_state_at(&self, hash: &Block::Hash, number: NumberFor<Block>) -> bool {
-		!self.storage.state_db.is_pruned(hash, number.saturated_into::<u64>())
+		if self.is_archive {
+			match self.blockchain.header(BlockId::Hash(hash.clone())) {
+				Ok(Some(header)) => {
+					state_machine::Storage::get(self.storage.as_ref(), &header.state_root(), (&[], None)).unwrap_or(None).is_some()
+				},
+				_ => false,
+			}
+		} else {
+			!self.storage.state_db.is_pruned(hash, number.saturated_into::<u64>())
+		}
 	}
 
 	fn destroy_state(&self, state: Self::State) -> ClientResult<()> {
@@ -1581,7 +1610,7 @@ mod tests {
 		};
 		let mut op = backend.begin_operation().unwrap();
 		backend.begin_state_operation(&mut op, block_id).unwrap();
-		op.set_block_data(header, None, None, NewBlockState::Best).unwrap();
+		op.set_block_data(header, Some(Vec::new()), None, NewBlockState::Best).unwrap();
 		op.update_changes_trie((changes_trie_update, ChangesTrieCacheAction::Clear)).unwrap();
 		backend.commit_operation(op).unwrap();
 

core/client/src/client.rs

@@ -838,15 +838,22 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 			finalized,
 			auxiliary,
 			fork_choice,
+			allow_missing_state,
 		} = import_block;
 
 		assert!(justification.is_some() && finalized || justification.is_none());
 
 		let parent_hash = header.parent_hash().clone();
+		let mut enact_state = true;
 
-		match self.backend.blockchain().status(BlockId::Hash(parent_hash))? {
-			blockchain::BlockStatus::InChain => {},
-			blockchain::BlockStatus::Unknown => return Ok(ImportResult::UnknownParent),
+		match self.block_status(&BlockId::Hash(parent_hash))? {
+			BlockStatus::Unknown => return Ok(ImportResult::UnknownParent),
+			BlockStatus::InChainWithState | BlockStatus::Queued => {},
+			BlockStatus::InChainPruned if allow_missing_state => {
+				enact_state = false;
+			},
+			BlockStatus::InChainPruned => return Ok(ImportResult::MissingState),
+			BlockStatus::KnownBad => return Ok(ImportResult::KnownBad),
 		}
 
 		let import_headers = if post_digests.is_empty() {
@@ -875,6 +882,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 			finalized,
 			auxiliary,
 			fork_choice,
+			enact_state,
 		);
 
 		if let Ok(ImportResult::Imported(ref aux)) = result {
@@ -902,6 +910,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		finalized: bool,
 		aux: Vec<(Vec<u8>, Option<Vec<u8>>)>,
 		fork_choice: ForkChoiceStrategy,
+		enact_state: bool,
 	) -> error::Result<ImportResult> where
 		E: CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
 	{
@@ -927,22 +936,39 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 			BlockOrigin::Genesis | BlockOrigin::NetworkInitialSync | BlockOrigin::File => false,
 		};
 
-		self.backend.begin_state_operation(&mut operation.op, BlockId::Hash(parent_hash))?;
+		let storage_changes = match &body {
+			Some(body) if enact_state => {
+				self.backend.begin_state_operation(&mut operation.op, BlockId::Hash(parent_hash))?;
 
-		// ensure parent block is finalized to maintain invariant that
-		// finality is called sequentially.
-		if finalized {
-			self.apply_finality_with_block_hash(operation, parent_hash, None, info.best_hash, make_notifications)?;
-		}
+				// ensure parent block is finalized to maintain invariant that
+				// finality is called sequentially.
+				if finalized {
+					self.apply_finality_with_block_hash(operation, parent_hash, None, info.best_hash, make_notifications)?;
+				}
 
-		// FIXME #1232: correct path logic for when to execute this function
-		let (storage_update, changes_update, storage_changes) = self.block_execution(
-			&operation.op,
-			&import_headers,
-			origin,
-			hash,
-			body.clone(),
-		)?;
+				// FIXME #1232: correct path logic for when to execute this function
+				let (storage_update, changes_update, storage_changes) = self.block_execution(
+					&operation.op,
+					&import_headers,
+					origin,
+					hash,
+					&body,
+				)?;
+
+				operation.op.update_cache(new_cache);
+				if let Some(storage_update) = storage_update {
+					operation.op.update_db_storage(storage_update)?;
+				}
+				if let Some(storage_changes) = storage_changes.clone() {
+					operation.op.update_storage(storage_changes.0, storage_changes.1)?;
+				}
+				if let Some(Some(changes_update)) = changes_update {
+					operation.op.update_changes_trie(changes_update)?;
+				}
+				storage_changes
+			},
+			_ => None,
+		};
 
 		let is_new_best = finalized || match fork_choice {
 			ForkChoiceStrategy::LongestChain => import_headers.post().number() > &info.best_number,
@@ -977,17 +1003,6 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 			leaf_state,
 		)?;
 
-		operation.op.update_cache(new_cache);
-		if let Some(storage_update) = storage_update {
-			operation.op.update_db_storage(storage_update)?;
-		}
-		if let Some(storage_changes) = storage_changes.clone() {
-			operation.op.update_storage(storage_changes.0, storage_changes.1)?;
-		}
-		if let Some(Some(changes_update)) = changes_update {
-			operation.op.update_changes_trie(changes_update)?;
-		}
-
 		operation.op.insert_aux(aux)?;
 
 		if make_notifications {
@@ -1014,7 +1029,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		import_headers: &PrePostHeader<Block::Header>,
 		origin: BlockOrigin,
 		hash: Block::Hash,
-		body: Option<Vec<Block::Extrinsic>>,
+		body: &[Block::Extrinsic],
 	) -> error::Result<(
 		Option<StorageUpdate<B, Block>>,
 		Option<Option<ChangesUpdate<Block>>>,
@@ -1052,7 +1067,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 
 				let encoded_block = <Block as BlockT>::encode_from(
 					import_headers.pre(),
-					&body.unwrap_or_default()
+					body,
 				);
 
 				let (_, storage_update, changes_update) = self.executor
@@ -1523,7 +1538,7 @@ impl<'a, B, E, Block, RA> consensus::BlockImport<Block> for &'a Client<B, E, Blo
 		&mut self,
 		block: BlockCheckParams<Block>,
 	) -> Result<ImportResult, Self::Error> {
-		let BlockCheckParams { hash, number, parent_hash } = block;
+		let BlockCheckParams { hash, number, parent_hash, allow_missing_state } = block;
 
 		if let Some(h) = self.fork_blocks.as_ref().and_then(|x| x.get(&number)) {
 			if &hash != h  {
@@ -1541,7 +1556,9 @@ impl<'a, B, E, Block, RA> consensus::BlockImport<Block> for &'a Client<B, E, Blo
 			.map_err(|e| ConsensusError::ClientImport(e.to_string()))?
 		{
 			BlockStatus::InChainWithState | BlockStatus::Queued => {},
-			BlockStatus::Unknown | BlockStatus::InChainPruned => return Ok(ImportResult::UnknownParent),
+			BlockStatus::Unknown => return Ok(ImportResult::UnknownParent),
+			BlockStatus::InChainPruned if allow_missing_state => {},
+			BlockStatus::InChainPruned => return Ok(ImportResult::MissingState),
 			BlockStatus::KnownBad => return Ok(ImportResult::KnownBad),
 		}
 
@@ -1553,7 +1570,6 @@ impl<'a, B, E, Block, RA> consensus::BlockImport<Block> for &'a Client<B, E, Blo
 			BlockStatus::KnownBad => return Ok(ImportResult::KnownBad),
 		}
 
-
 		Ok(ImportResult::imported(false))
 	}
 }

core/consensus/aura/src/lib.rs

@@ -267,6 +267,7 @@ impl<H, B, C, E, I, P, Error, SO> slots::SimpleSlotWorker<B> for AuraWorker<C, E
 				finalized: false,
 				auxiliary: Vec::new(),
 				fork_choice: ForkChoiceStrategy::LongestChain,
+				allow_missing_state: false,
 			}
 		})
 	}
@@ -570,6 +571,7 @@ impl<B: BlockT, C, P, T> Verifier<B> for AuraVerifier<C, P, T> where
 					justification,
 					auxiliary: Vec::new(),
 					fork_choice: ForkChoiceStrategy::LongestChain,
+					allow_missing_state: false,
 				};
 
 				Ok((block_import_params, maybe_keys))

core/consensus/babe/src/lib.rs

@@ -430,6 +430,7 @@ impl<B, C, E, I, Error, SO> slots::SimpleSlotWorker<B> for BabeWorker<B, C, E, I
 				// option to specify one.
 				// https://github.com/paritytech/substrate/issues/3623
 				fork_choice: ForkChoiceStrategy::LongestChain,
+				allow_missing_state: false,
 			}
 		})
 	}
@@ -741,6 +742,7 @@ impl<B, E, Block, RA, PRA> Verifier<Block> for BabeVerifier<B, E, Block, RA, PRA
 					// option to specify one.
 					// https://github.com/paritytech/substrate/issues/3623
 					fork_choice: ForkChoiceStrategy::LongestChain,
+					allow_missing_state: false,
 				};
 
 				Ok((block_import_params, Default::default()))

core/consensus/babe/src/tests.rs

@@ -578,6 +578,7 @@ fn propose_and_import_block(
 			finalized: false,
 			auxiliary: Vec::new(),
 			fork_choice: ForkChoiceStrategy::LongestChain,
+			allow_missing_state: false,
 		},
 		Default::default(),
 	).unwrap();

core/consensus/common/src/block_import.rs

@@ -35,11 +35,15 @@ pub enum ImportResult {
 	KnownBad,
 	/// Block parent is not in the chain.
 	UnknownParent,
+	/// Parent state is missing.
+	MissingState,
 }
 
 /// Auxiliary data associated with an imported block result.
 #[derive(Debug, Default, PartialEq, Eq)]
 pub struct ImportedAux {
+	/// Only the header has been imported. Block body verification was skipped.
+	pub header_only: bool,
 	/// Clear all pending justification requests.
 	pub clear_justification_requests: bool,
 	/// Request a justification for the given block.
@@ -98,6 +102,8 @@ pub struct BlockCheckParams<Block: BlockT> {
 	pub number: NumberFor<Block>,
 	/// Parent hash of the block that we verify.
 	pub parent_hash: Block::Hash,
+	/// Allow importing the block skipping state verification if parent state is missing.
+	pub allow_missing_state: bool,
 }
 
 /// Data required to import a Block.
@@ -133,6 +139,8 @@ pub struct BlockImportParams<Block: BlockT> {
 	/// Fork choice strategy of this import. This should only be set by a
 	/// synchronous import, otherwise it may race against other imports.
 	pub fork_choice: ForkChoiceStrategy,
+	/// Allow importing the block skipping state verification if parent state is missing.
+	pub allow_missing_state: bool,
 }
 
 impl<Block: BlockT> BlockImportParams<Block> {