diff --git a/benches/benches/block_target_gas.rs b/benches/benches/block_target_gas.rs index a96cf389341..e52e9882d29 100644 --- a/benches/benches/block_target_gas.rs +++ b/benches/benches/block_target_gas.rs @@ -256,7 +256,7 @@ fn service_with_many_contracts( .build() .unwrap(); let _drop = rt.enter(); - let mut database = Database::rocksdb(); + let mut database = Database::rocksdb_temp(); let mut config = Config::local_node(); config .chain_conf diff --git a/crates/database/src/lib.rs b/crates/database/src/lib.rs index 578f8c48010..e1b1b940048 100644 --- a/crates/database/src/lib.rs +++ b/crates/database/src/lib.rs @@ -21,12 +21,6 @@ pub enum Error { /// Error occurred during serialization or deserialization of the entity. #[display(fmt = "error performing serialization or deserialization")] Codec, - /// Chain can be initialized once. - #[display(fmt = "Failed to initialize chain")] - ChainAlreadyInitialized, - /// Chain should be initialized before usage. - #[display(fmt = "Chain is not yet initialized")] - ChainUninitialized, /// The version of database or data is invalid (possibly not migrated). #[display( fmt = "Invalid database version, expected {expected:#x}, found {found:#x}" @@ -37,6 +31,33 @@ pub enum Error { /// the database version expected by this build of fuel-core expected: u32, }, + /// Multiple heights found in the commit to teh database. + #[display(fmt = "Multiple heights found in the commit {heights:?}")] + MultipleHeightsInCommit { + /// List of heights found in the commit. + heights: Vec, + }, + /// Failed to advance the height. + #[display(fmt = "Failed to advance the height")] + FailedToAdvanceHeight, + /// The new and old heights are not linked. + #[display( + fmt = "New and old heights are not linked: prev_height: {prev_height:#x}, new_height: {new_height:#x}" + )] + HeightsAreNotLinked { + /// The old height. + prev_height: u64, + /// The new height. + new_height: u64, + }, + /// The new height is not found, but the old height is set. + #[display( + fmt = "The new height is not found, but the old height is set: prev_height: {prev_height:#x}" + )] + NewHeightIsNotSet { + /// The old height known by the database. + prev_height: u64, + }, /// Not related to database error. #[from] diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index 964f0b1e7cc..75578ed96ce 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -7,10 +7,6 @@ use crate::database::{ Database, }; use fuel_core_storage::Result as StorageResult; -use fuel_core_types::{ - blockchain::primitives::DaBlockHeight, - fuel_types::BlockHeight, -}; /// A database that combines the on-chain, off-chain and relayer databases into one entity. #[derive(Default, Clone)] @@ -57,14 +53,10 @@ impl CombinedDatabase { ) } - pub fn init( - &mut self, - block_height: &BlockHeight, - da_block_height: &DaBlockHeight, - ) -> StorageResult<()> { - self.on_chain.init(block_height)?; - self.off_chain.init(block_height)?; - self.relayer.init(da_block_height)?; + pub fn check_version(&self) -> StorageResult<()> { + self.on_chain.check_version()?; + self.off_chain.check_version()?; + self.relayer.check_version()?; Ok(()) } diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 7f2a6150d83..8ef2729195f 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -1,12 +1,19 @@ use crate::{ - database::database_description::{ - off_chain::OffChain, - on_chain::OnChain, - relayer::Relayer, - DatabaseDescription, + database::{ + database_description::{ + off_chain::OffChain, + on_chain::OnChain, + relayer::Relayer, + DatabaseDescription, + DatabaseMetadata, + }, + metadata::MetadataTable, + Error as DatabaseError, }, + graphql_api::storage::blocks::FuelBlockIdsToHeights, state::{ in_memory::memory_store::MemoryStore, + ChangesIterator, DataSource, }, }; @@ -21,26 +28,39 @@ use fuel_core_storage::{ BoxedIter, IterDirection, IterableStore, + IteratorOverTable, }, kv_store::{ KVItem, KeyValueInspect, Value, }, + not_found, + tables::FuelBlocks, transactional::{ AtomicView, Changes, + ConflictPolicy, Modifiable, + StorageTransaction, }, + Error as StorageError, Result as StorageResult, + StorageAsMut, + StorageMutate, }; use fuel_core_types::{ blockchain::primitives::DaBlockHeight, + fuel_merkle::storage::StorageInspect, fuel_types::BlockHeight, }; +use itertools::Itertools; use std::{ fmt::Debug, - sync::Arc, + sync::{ + Arc, + Mutex, + }, }; pub use fuel_core_database::Error; @@ -70,15 +90,27 @@ pub struct Database where Description: DatabaseDescription, { + height: Arc>>, data: DataSource, } impl Database where Description: DatabaseDescription, + Self: StorageInspect, Error = StorageError>, { pub fn new(data_source: DataSource) -> Self { - Self { data: data_source } + let mut database = Self { + height: Arc::new(Mutex::new(None)), + data: data_source, + }; + let height = database + .latest_height() + .expect("Failed to get latest height during creation of the database"); + + database.height = Arc::new(Mutex::new(height)); + + database } #[cfg(feature = "rocksdb")] @@ -86,19 +118,30 @@ where use anyhow::Context; let db = RocksDb::::default_open(path, capacity.into()).map_err(Into::::into).context("Failed to open rocksdb, you may need to wipe a pre-existing incompatible db `rm -rf ~/.fuel/db`")?; - Ok(Database { data: Arc::new(db) }) + Ok(Database::new(Arc::new(db))) } +} +impl Database +where + Description: DatabaseDescription, +{ pub fn in_memory() -> Self { let data = Arc::>::new(MemoryStore::default()); - Self { data } + Self { + height: Arc::new(Mutex::new(None)), + data, + } } #[cfg(feature = "rocksdb")] - pub fn rocksdb() -> Self { + pub fn rocksdb_temp() -> Self { let data = Arc::>::new(RocksDb::default_open_temp(None).unwrap()); - Self { data } + Self { + height: Arc::new(Mutex::new(None)), + data, + } } } @@ -134,15 +177,6 @@ where } } -impl Modifiable for Database -where - Description: DatabaseDescription, -{ - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - self.data.as_ref().commit_changes(changes) - } -} - impl IterableStore for Database where Description: DatabaseDescription, @@ -160,24 +194,6 @@ where } } -impl AsRef> for Database -where - Description: DatabaseDescription, -{ - fn as_ref(&self) -> &Database { - self - } -} - -impl AsMut> for Database -where - Description: DatabaseDescription, -{ - fn as_mut(&mut self) -> &mut Database { - self - } -} - /// Construct an ephemeral database /// uses rocksdb when rocksdb features are enabled /// uses in-memory when rocksdb features are disabled @@ -192,7 +208,7 @@ where } #[cfg(feature = "rocksdb")] { - Self::rocksdb() + Self::rocksdb_temp() } } } @@ -213,7 +229,8 @@ impl ChainConfigDb for Database { } fn get_block_height(&self) -> StorageResult { - self.latest_height() + self.latest_height()? + .ok_or(not_found!("Block height not found")) } } @@ -222,11 +239,8 @@ impl AtomicView for Database { type Height = BlockHeight; - fn latest_height(&self) -> BlockHeight { - // TODO: The database should track the latest height inside of the database object - // instead of fetching it from the `FuelBlocks` table. As a temporary solution, - // fetch it from the table for now. - self.latest_height().unwrap_or_default() + fn latest_height(&self) -> Option { + *self.height.lock().expect("poisoned") } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -245,11 +259,8 @@ impl AtomicView for Database { type Height = BlockHeight; - fn latest_height(&self) -> BlockHeight { - // TODO: The database should track the latest height inside of the database object - // instead of fetching it from the `FuelBlocks` table. As a temporary solution, - // fetch it from the table for now. - self.latest_height().unwrap_or_default() + fn latest_height(&self) -> Option { + *self.height.lock().expect("poisoned") } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -267,20 +278,8 @@ impl AtomicView for Database { type View = Self; type Height = DaBlockHeight; - fn latest_height(&self) -> Self::Height { - #[cfg(feature = "relayer")] - { - use fuel_core_relayer::ports::RelayerDb; - // TODO: The database should track the latest da height inside of the database object - // instead of fetching it from the `RelayerMetadata` table. As a temporary solution, - // fetch it from the table for now. - // https://github.com/FuelLabs/fuel-core/issues/1589 - self.get_finalized_da_height().unwrap_or_default() - } - #[cfg(not(feature = "relayer"))] - { - DaBlockHeight(0) - } + fn latest_height(&self) -> Option { + *self.height.lock().expect("poisoned") } fn view_at(&self, _: &Self::Height) -> StorageResult { @@ -292,6 +291,176 @@ impl AtomicView for Database { } } +impl Modifiable for Database { + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + commit_changes_with_height_update(self, changes, |iter| { + iter.iter_all::(Some(IterDirection::Reverse)) + .map(|result| result.map(|(height, _)| height)) + .try_collect() + }) + } +} + +impl Modifiable for Database { + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + commit_changes_with_height_update(self, changes, |iter| { + iter.iter_all::(Some(IterDirection::Reverse)) + .map(|result| result.map(|(_, height)| height)) + .try_collect() + }) + } +} + +#[cfg(feature = "relayer")] +impl Modifiable for Database { + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + commit_changes_with_height_update(self, changes, |iter| { + iter.iter_all::(Some( + IterDirection::Reverse, + )) + .map(|result| result.map(|(height, _)| height)) + .try_collect() + }) + } +} + +#[cfg(not(feature = "relayer"))] +impl Modifiable for Database { + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + commit_changes_with_height_update(self, changes, |_| Ok(vec![])) + } +} + +trait DatabaseHeight: Sized { + fn as_u64(&self) -> u64; + + fn advance_height(&self) -> Option; +} + +impl DatabaseHeight for BlockHeight { + fn as_u64(&self) -> u64 { + let height: u32 = (*self).into(); + height as u64 + } + + fn advance_height(&self) -> Option { + self.succ() + } +} + +impl DatabaseHeight for DaBlockHeight { + fn as_u64(&self) -> u64 { + self.0 + } + + fn advance_height(&self) -> Option { + self.0.checked_add(1).map(Into::into) + } +} + +fn commit_changes_with_height_update( + database: &mut Database, + changes: Changes, + heights_lookup: impl Fn( + &ChangesIterator, + ) -> StorageResult>, +) -> StorageResult<()> +where + Description: DatabaseDescription, + Description::Height: Debug + PartialOrd + DatabaseHeight, + for<'a> StorageTransaction<&'a &'a mut Database>: + StorageMutate, Error = StorageError>, +{ + // Gets the all new heights from the `changes` + let iterator = ChangesIterator::::new(&changes); + let new_heights = heights_lookup(&iterator)?; + + // Changes for each block should be committed separately. + // If we have more than one height, it means we are mixing commits + // for several heights in one butch - return error in this case. + if new_heights.len() > 1 { + return Err(DatabaseError::MultipleHeightsInCommit { + heights: new_heights.iter().map(DatabaseHeight::as_u64).collect(), + } + .into()); + } + + let new_height = new_heights.into_iter().last(); + let prev_height = *database.height.lock().expect("poisoned"); + + match (prev_height, new_height) { + (None, None) => { + // We are inside the regenesis process if the old and new heights are not set. + // In this case, we continue to commit until we discover a new height. + // This height will be the start of the database. + } + (Some(prev_height), Some(new_height)) => { + // Each new commit should be linked to the previous commit to create a monotonically growing database. + + let next_expected_height = prev_height + .advance_height() + .ok_or(DatabaseError::FailedToAdvanceHeight)?; + + // TODO: After https://github.com/FuelLabs/fuel-core/issues/451 + // we can replace `next_expected_height > new_height` with `next_expected_height != new_height`. + if next_expected_height > new_height { + return Err(DatabaseError::HeightsAreNotLinked { + prev_height: prev_height.as_u64(), + new_height: new_height.as_u64(), + } + .into()); + } + } + (None, Some(_)) => { + // The new height is finally found; starting at this point, + // all next commits should be linked(the height should increase each time by one). + } + (Some(prev_height), None) => { + // In production, we shouldn't have cases where we call `commit_chagnes` with intermediate changes. + // The commit always should contain all data for the corresponding height. + return Err(DatabaseError::NewHeightIsNotSet { + prev_height: prev_height.as_u64(), + } + .into()); + } + }; + + let updated_changes = if let Some(new_height) = new_height { + // We want to update the metadata table to include a new height. + // For that, we are building a new storage transaction around `changes`. + // Modifying this transaction will include all required updates into the `changes`. + let mut transaction = StorageTransaction::transaction( + &database, + ConflictPolicy::Overwrite, + changes, + ); + transaction + .storage_as_mut::>() + .insert( + &(), + &DatabaseMetadata::V1 { + version: Description::version(), + height: new_height, + }, + )?; + + transaction.into_changes() + } else { + changes + }; + + let mut guard = database.height.lock().expect("poisoned"); + database + .data + .as_ref() + .commit_changes(new_height, updated_changes)?; + + // Update the block height + *guard = new_height; + + Ok(()) +} + #[cfg(feature = "rocksdb")] pub fn convert_to_rocksdb_direction(direction: IterDirection) -> rocksdb::Direction { match direction { @@ -302,11 +471,14 @@ pub fn convert_to_rocksdb_direction(direction: IterDirection) -> rocksdb::Direct #[cfg(test)] mod tests { - use crate::database::database_description::{ - off_chain::OffChain, - on_chain::OnChain, - relayer::Relayer, - DatabaseDescription, + use super::*; + use crate::database::{ + database_description::DatabaseDescription, + Database, + }; + use fuel_core_storage::{ + tables::FuelBlocks, + StorageAsMut, }; fn column_keys_not_exceed_count() @@ -321,18 +493,486 @@ mod tests { } } - #[test] - fn column_keys_not_exceed_count_test_on_chain() { - column_keys_not_exceed_count::(); + mod on_chain { + use super::*; + use crate::database::{ + database_description::on_chain::OnChain, + DatabaseHeight, + }; + use fuel_core_storage::{ + tables::Coins, + transactional::WriteTransaction, + }; + use fuel_core_types::blockchain::block::CompressedBlock; + + #[test] + fn column_keys_not_exceed_count_test() { + column_keys_not_exceed_count::(); + } + + #[test] + fn database_advances_with_a_new_block() { + // Given + let mut database = Database::::default(); + assert_eq!(database.latest_height().unwrap(), None); + + // When + let advanced_height = 1.into(); + database + .storage_as_mut::() + .insert(&advanced_height, &CompressedBlock::default()) + .unwrap(); + + // Then + assert_eq!(database.latest_height().unwrap(), Some(advanced_height)); + } + + #[test] + fn database_not_advances_without_block() { + // Given + let mut database = Database::::default(); + assert_eq!(database.latest_height().unwrap(), None); + + // When + database + .storage_as_mut::() + .insert(&Default::default(), &Default::default()) + .unwrap(); + + // Then + assert_eq!(database.latest_height().unwrap(), None); + } + + #[test] + fn database_advances_with_linked_blocks() { + // Given + let mut database = Database::::default(); + let starting_height = 1.into(); + database + .storage_as_mut::() + .insert(&starting_height, &CompressedBlock::default()) + .unwrap(); + assert_eq!(database.latest_height().unwrap(), Some(starting_height)); + + // When + let next_height = starting_height.advance_height().unwrap(); + database + .storage_as_mut::() + .insert(&next_height, &CompressedBlock::default()) + .unwrap(); + + // Then + assert_eq!(database.latest_height().unwrap(), Some(next_height)); + } + + #[test] + fn database_fails_with_unlinked_blocks() { + // Given + let mut database = Database::::default(); + let starting_height = 1.into(); + database + .storage_as_mut::() + .insert(&starting_height, &CompressedBlock::default()) + .unwrap(); + + // When + let prev_height = 0.into(); + let result = database + .storage_as_mut::() + .insert(&prev_height, &CompressedBlock::default()); + + // Then + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + StorageError::from(DatabaseError::HeightsAreNotLinked { + prev_height: 1, + new_height: 0 + }) + .to_string() + ); + } + + #[test] + fn database_fails_with_non_advancing_commit() { + // Given + let mut database = Database::::default(); + let starting_height = 1.into(); + database + .storage_as_mut::() + .insert(&starting_height, &CompressedBlock::default()) + .unwrap(); + + // When + let result = database + .storage_as_mut::() + .insert(&Default::default(), &Default::default()); + + // Then + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + StorageError::from(DatabaseError::NewHeightIsNotSet { prev_height: 1 }) + .to_string() + ); + } + + #[test] + fn database_fails_when_commit_with_several_blocks() { + let mut database = Database::::default(); + let starting_height = 1.into(); + database + .storage_as_mut::() + .insert(&starting_height, &CompressedBlock::default()) + .unwrap(); + + // Given + let mut transaction = database.write_transaction(); + let next_height = starting_height.advance_height().unwrap(); + let next_next_height = next_height.advance_height().unwrap(); + transaction + .storage_as_mut::() + .insert(&next_height, &CompressedBlock::default()) + .unwrap(); + transaction + .storage_as_mut::() + .insert(&next_next_height, &CompressedBlock::default()) + .unwrap(); + + // When + let result = transaction.commit(); + + // Then + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + StorageError::from(DatabaseError::MultipleHeightsInCommit { + heights: vec![3, 2] + }) + .to_string() + ); + } } - #[test] - fn column_keys_not_exceed_count_test_off_chain() { - column_keys_not_exceed_count::(); + mod off_chain { + use super::*; + use crate::{ + database::{ + database_description::off_chain::OffChain, + DatabaseHeight, + }, + graphql_api::storage::messages::OwnedMessageIds, + }; + use fuel_core_storage::transactional::WriteTransaction; + + #[test] + fn column_keys_not_exceed_count_test() { + column_keys_not_exceed_count::(); + } + + #[test] + fn database_advances_with_a_new_block() { + // Given + let mut database = Database::::default(); + assert_eq!(database.latest_height().unwrap(), None); + + // When + let advanced_height = 1.into(); + database + .storage_as_mut::() + .insert(&Default::default(), &advanced_height) + .unwrap(); + + // Then + assert_eq!(database.latest_height().unwrap(), Some(advanced_height)); + } + + #[test] + fn database_not_advances_without_block() { + // Given + let mut database = Database::::default(); + assert_eq!(database.latest_height().unwrap(), None); + + // When + database + .storage_as_mut::() + .insert(&Default::default(), &Default::default()) + .unwrap(); + + // Then + assert_eq!(database.latest_height().unwrap(), None); + } + + #[test] + fn database_advances_with_linked_blocks() { + // Given + let mut database = Database::::default(); + let starting_height = 1.into(); + database + .storage_as_mut::() + .insert(&Default::default(), &starting_height) + .unwrap(); + assert_eq!(database.latest_height().unwrap(), Some(starting_height)); + + // When + let next_height = starting_height.advance_height().unwrap(); + database + .storage_as_mut::() + .insert(&Default::default(), &next_height) + .unwrap(); + + // Then + assert_eq!(database.latest_height().unwrap(), Some(next_height)); + } + + #[test] + fn database_fails_with_unlinked_blocks() { + // Given + let mut database = Database::::default(); + let starting_height = 1.into(); + database + .storage_as_mut::() + .insert(&Default::default(), &starting_height) + .unwrap(); + + // When + let prev_height = 0.into(); + let result = database + .storage_as_mut::() + .insert(&Default::default(), &prev_height); + + // Then + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + StorageError::from(DatabaseError::HeightsAreNotLinked { + prev_height: 1, + new_height: 0 + }) + .to_string() + ); + } + + #[test] + fn database_fails_with_non_advancing_commit() { + // Given + let mut database = Database::::default(); + let starting_height = 1.into(); + database + .storage_as_mut::() + .insert(&Default::default(), &starting_height) + .unwrap(); + + // When + let result = database + .storage_as_mut::() + .insert(&Default::default(), &Default::default()); + + // Then + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + StorageError::from(DatabaseError::NewHeightIsNotSet { prev_height: 1 }) + .to_string() + ); + } + + #[test] + fn database_fails_when_commit_with_several_blocks() { + let mut database = Database::::default(); + let starting_height = 1.into(); + database + .storage_as_mut::() + .insert(&Default::default(), &starting_height) + .unwrap(); + + // Given + let mut transaction = database.write_transaction(); + let next_height = starting_height.advance_height().unwrap(); + let next_next_height = next_height.advance_height().unwrap(); + transaction + .storage_as_mut::() + .insert(&[1; 32].into(), &next_height) + .unwrap(); + transaction + .storage_as_mut::() + .insert(&[2; 32].into(), &next_next_height) + .unwrap(); + + // When + let result = transaction.commit(); + + // Then + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + StorageError::from(DatabaseError::MultipleHeightsInCommit { + heights: vec![3, 2] + }) + .to_string() + ); + } } - #[test] - fn column_keys_not_exceed_count_test_relayer() { - column_keys_not_exceed_count::(); + #[cfg(feature = "relayer")] + mod relayer { + use super::*; + use crate::database::{ + database_description::relayer::Relayer, + DatabaseHeight, + }; + use fuel_core_relayer::storage::{ + DaHeightTable, + EventsHistory, + }; + use fuel_core_storage::transactional::WriteTransaction; + + #[test] + fn column_keys_not_exceed_count_test() { + column_keys_not_exceed_count::(); + } + + #[test] + fn database_advances_with_a_new_block() { + // Given + let mut database = Database::::default(); + assert_eq!(database.latest_height().unwrap(), None); + + // When + let advanced_height = 1u64.into(); + database + .storage_as_mut::() + .insert(&advanced_height, &[]) + .unwrap(); + + // Then + assert_eq!(database.latest_height().unwrap(), Some(advanced_height)); + } + + #[test] + fn database_not_advances_without_block() { + // Given + let mut database = Database::::default(); + assert_eq!(database.latest_height().unwrap(), None); + + // When + database + .storage_as_mut::() + .insert(&Default::default(), &Default::default()) + .unwrap(); + + // Then + assert_eq!(database.latest_height().unwrap(), None); + } + + #[test] + fn database_advances_with_linked_blocks() { + // Given + let mut database = Database::::default(); + let starting_height = 1u64.into(); + database + .storage_as_mut::() + .insert(&starting_height, &[]) + .unwrap(); + assert_eq!(database.latest_height().unwrap(), Some(starting_height)); + + // When + let next_height = starting_height.advance_height().unwrap(); + database + .storage_as_mut::() + .insert(&next_height, &[]) + .unwrap(); + + // Then + assert_eq!(database.latest_height().unwrap(), Some(next_height)); + } + + #[test] + fn database_fails_with_unlinked_blocks() { + // Given + let mut database = Database::::default(); + let starting_height = 1u64.into(); + database + .storage_as_mut::() + .insert(&starting_height, &[]) + .unwrap(); + + // When + let prev_height = 0u64.into(); + let result = database + .storage_as_mut::() + .insert(&prev_height, &[]); + + // Then + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + StorageError::from(DatabaseError::HeightsAreNotLinked { + prev_height: 1, + new_height: 0 + }) + .to_string() + ); + } + + #[test] + fn database_fails_with_non_advancing_commit() { + // Given + let mut database = Database::::default(); + let starting_height = 1u64.into(); + database + .storage_as_mut::() + .insert(&starting_height, &[]) + .unwrap(); + + // When + let result = database + .storage_as_mut::() + .insert(&Default::default(), &Default::default()); + + // Then + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + StorageError::from(DatabaseError::NewHeightIsNotSet { prev_height: 1 }) + .to_string() + ); + } + + #[test] + fn database_fails_when_commit_with_several_blocks() { + let mut database = Database::::default(); + let starting_height = 1u64.into(); + database + .storage_as_mut::() + .insert(&starting_height, &[]) + .unwrap(); + + // Given + let mut transaction = database.write_transaction(); + let next_height = starting_height.advance_height().unwrap(); + let next_next_height = next_height.advance_height().unwrap(); + transaction + .storage_as_mut::() + .insert(&next_height, &[]) + .unwrap(); + transaction + .storage_as_mut::() + .insert(&next_next_height, &[]) + .unwrap(); + + // When + let result = transaction.commit(); + + // Then + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + StorageError::from(DatabaseError::MultipleHeightsInCommit { + heights: vec![3, 2] + }) + .to_string() + ); + } } } diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 8f2aefec465..f62c6a11695 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -10,7 +10,7 @@ pub trait DatabaseDescription: 'static + Clone + Debug + Send + Sync { /// The type of the column used by the database. type Column: StorageColumn + strum::EnumCount + enum_iterator::Sequence; /// The type of the height of the database used to track commits. - type Height: Copy; + type Height: Default + Copy; /// Returns the expected version of the database. fn version() -> u32; diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index a4b819ccd02..d9f8672b073 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -9,14 +9,13 @@ use crate::database::{ use fuel_core_storage::{ blueprint::plain::Plain, codec::postcard::Postcard, - not_found, structured_storage::TableWithBlueprint, Error as StorageError, Mappable, Result as StorageResult, StorageAsRef, + StorageInspect, }; -use fuel_core_types::fuel_merkle::storage::StorageMutate; /// The table that stores all metadata about the database. pub struct MetadataTable(core::marker::PhantomData); @@ -46,33 +45,14 @@ where impl Database where Description: DatabaseDescription, - Self: StorageMutate, Error = StorageError>, + Self: StorageInspect, Error = StorageError>, { - /// Ensures the database is initialized and that the database version is correct - pub fn init(&mut self, height: &Description::Height) -> StorageResult<()> { - use fuel_core_storage::StorageAsMut; - - if !self - .storage::>() - .contains_key(&())? - { - let old = self.storage::>().insert( - &(), - &DatabaseMetadata::V1 { - version: Description::version(), - height: *height, - }, - )?; - - if old.is_some() { - return Err(DatabaseError::ChainAlreadyInitialized.into()) - } - } - - let metadata = self - .storage::>() - .get(&())? - .expect("We checked its existence above"); + /// Ensures the version is correct. + pub fn check_version(&self) -> StorageResult<()> { + let Some(metadata) = self.storage::>().get(&())? + else { + return Ok(()); + }; if metadata.version() != Description::version() { return Err(DatabaseError::InvalidDatabaseVersion { @@ -85,11 +65,11 @@ where Ok(()) } - pub fn latest_height(&self) -> StorageResult { + pub fn latest_height(&self) -> StorageResult> { let metadata = self.storage::>().get(&())?; - let metadata = metadata.ok_or(not_found!(MetadataTable))?; + let metadata = metadata.map(|metadata| *metadata.height()); - Ok(*metadata.height()) + Ok(metadata) } } diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index 381a3a44655..a7241146828 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -51,12 +51,14 @@ where } } +// TODO: After https://github.com/FuelLabs/fuel-vm/pull/679 implement it only for `feature = "test-helpers"` impl StorageMutate for Database where Description: DatabaseDescription, M: Mappable, for<'a> StructuredStorage<&'a Self>: StorageInspect, for<'a> StorageTransaction<&'a Self>: StorageMutate, + Self: Modifiable, { fn insert( &mut self, @@ -102,6 +104,7 @@ where M: Mappable, for<'a> StructuredStorage<&'a Self>: StorageInspect, for<'a> StorageTransaction<&'a Self>: MerkleRootStorage, + Self: Modifiable, { fn root(&self, key: &Key) -> StorageResult { // TODO: Use `StructuredStorage` instead of `StorageTransaction` https://github.com/FuelLabs/fuel-vm/pull/679 @@ -130,6 +133,7 @@ where M: Mappable, for<'a> StructuredStorage<&'a Self>: StorageInspect, for<'a> StorageTransaction<&'a Self>: StorageBatchMutate, + Self: Modifiable, { fn init_storage<'a, Iter>(&mut self, set: Iter) -> StorageResult<()> where diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index 1d65e11df97..c0a411c26bd 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -146,8 +146,8 @@ mod tests { type View = Self; type Height = DaBlockHeight; - fn latest_height(&self) -> Self::Height { - 0u64.into() + fn latest_height(&self) -> Option { + Some(0u64.into()) } fn view_at(&self, _: &Self::Height) -> StorageResult { @@ -2807,9 +2807,12 @@ mod tests { #[cfg(feature = "relayer")] mod relayer { use super::*; - use crate::database::database_description::{ - on_chain::OnChain, - relayer::Relayer, + use crate::{ + database::database_description::{ + on_chain::OnChain, + relayer::Relayer, + }, + state::ChangesIterator, }; use fuel_core_relayer::storage::EventsHistory; use fuel_core_storage::{ @@ -2935,15 +2938,17 @@ mod tests { assert_eq!(on_chain_db.iter_all::(None).count(), 0); // When - let mut producer = create_relayer_executor(on_chain_db, relayer_db); + let producer = create_relayer_executor(on_chain_db, relayer_db); let block = test_block(block_height.into(), block_da_height.into(), 0); - let result = producer.execute_and_commit( - ExecutionTypes::Production(block.into()), - Default::default(), - )?; + let (result, changes) = producer + .execute_without_commit( + ExecutionTypes::Production(block.into()), + Default::default(), + )? + .into(); // Then - let view = producer.database_view_provider.latest_view(); + let view = ChangesIterator::::new(&changes); assert_eq!( view.iter_all::(None).count() as u64, block_da_height - genesis_da_height @@ -2978,17 +2983,18 @@ mod tests { assert_eq!(on_chain_db.iter_all::(None).count(), 0); // When - let mut producer = create_relayer_executor(on_chain_db, relayer_db); + let producer = create_relayer_executor(on_chain_db, relayer_db); let block = test_block(block_height.into(), block_da_height.into(), 10); - let result = producer - .execute_and_commit( + let (result, changes) = producer + .execute_without_commit( ExecutionTypes::Production(block.into()), Default::default(), ) - .unwrap(); + .unwrap() + .into(); // Then - let view = producer.database_view_provider.latest_view(); + let view = ChangesIterator::::new(&changes); assert!(result.skipped_transactions.is_empty()); assert_eq!(view.iter_all::(None).count() as u64, 0); } @@ -3024,16 +3030,17 @@ mod tests { // When let mut block = test_block(block_height.into(), block_da_height.into(), 0); *block.transactions_mut() = vec![tx]; - let mut producer = create_relayer_executor(on_chain_db, relayer_db); - let result = producer - .execute_and_commit( + let producer = create_relayer_executor(on_chain_db, relayer_db); + let (result, changes) = producer + .execute_without_commit( ExecutionTypes::Production(block.into()), Default::default(), ) - .unwrap(); + .unwrap() + .into(); // Then - let view = producer.database_view_provider.latest_view(); + let view = ChangesIterator::::new(&changes); assert!(result.skipped_transactions.is_empty()); assert_eq!(view.iter_all::(None).count() as u64, 0); // Message added during this block immediately became spent. diff --git a/crates/fuel-core/src/graphql_api/database/arc_wrapper.rs b/crates/fuel-core/src/graphql_api/database/arc_wrapper.rs index a86043bd80a..5355bde48d8 100644 --- a/crates/fuel-core/src/graphql_api/database/arc_wrapper.rs +++ b/crates/fuel-core/src/graphql_api/database/arc_wrapper.rs @@ -38,7 +38,7 @@ where type View = OnChainView; type Height = Height; - fn latest_height(&self) -> Self::Height { + fn latest_height(&self) -> Option { self.inner.latest_height() } @@ -60,7 +60,7 @@ where type View = OffChainView; type Height = Height; - fn latest_height(&self) -> Self::Height { + fn latest_height(&self) -> Option { self.inner.latest_height() } diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index c9aec7dd309..64708fdbcfa 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -213,17 +213,10 @@ pub struct Task { impl Task { /// Private inner method for initializing the fuel service task - pub fn new(mut database: CombinedDatabase, config: Config) -> anyhow::Result { + pub fn new(database: CombinedDatabase, config: Config) -> anyhow::Result { // initialize state tracing::info!("Initializing database"); - let block_height = config - .chain_conf - .initial_state - .as_ref() - .and_then(|state| state.height) - .unwrap_or_default(); - let da_block_height = 0u64.into(); - database.init(&block_height, &da_block_height)?; + database.check_version()?; // initialize sub services tracing::info!("Initializing sub services"); diff --git a/crates/fuel-core/src/service/genesis.rs b/crates/fuel-core/src/service/genesis.rs index c79d993a88e..5e136e3c03c 100644 --- a/crates/fuel-core/src/service/genesis.rs +++ b/crates/fuel-core/src/service/genesis.rs @@ -439,6 +439,7 @@ mod tests { assert_eq!( test_height, db.latest_height() + .unwrap() .expect("Expected a block height to be set") ) } diff --git a/crates/fuel-core/src/state.rs b/crates/fuel-core/src/state.rs index ec3a289efbf..a707ba30b27 100644 --- a/crates/fuel-core/src/state.rs +++ b/crates/fuel-core/src/state.rs @@ -29,20 +29,25 @@ pub mod rocks_db; pub type DataSource where Description: DatabaseDescription, -= Arc>; += Arc>; -pub trait TransactableStorage: IterableStore + Debug + Send + Sync { +pub trait TransactableStorage: IterableStore + Debug + Send + Sync { /// Commits the changes into the storage. - fn commit_changes(&self, changes: Changes) -> StorageResult<()>; + fn commit_changes( + &self, + height: Option, + changes: Changes, + ) -> StorageResult<()>; } // It is used only to allow conversion of the `StorageTransaction` into the `DataSource`. #[cfg(feature = "test-helpers")] -impl TransactableStorage for fuel_core_storage::transactional::StorageTransaction +impl TransactableStorage + for fuel_core_storage::transactional::StorageTransaction where S: IterableStore + Debug + Send + Sync, { - fn commit_changes(&self, _: Changes) -> StorageResult<()> { + fn commit_changes(&self, _: Option, _: Changes) -> StorageResult<()> { unimplemented!() } } @@ -53,6 +58,16 @@ pub struct ChangesIterator<'a, Description> { _marker: core::marker::PhantomData, } +impl<'a, Description> ChangesIterator<'a, Description> { + /// Creates a new instance of the `ChangesIterator`. + pub fn new(changes: &'a Changes) -> Self { + Self { + changes, + _marker: Default::default(), + } + } +} + impl<'a, Description> KeyValueInspect for ChangesIterator<'a, Description> where Description: DatabaseDescription, diff --git a/crates/fuel-core/src/state/in_memory/memory_store.rs b/crates/fuel-core/src/state/in_memory/memory_store.rs index 200d0982020..f02b06eee1e 100644 --- a/crates/fuel-core/src/state/in_memory/memory_store.rs +++ b/crates/fuel-core/src/state/in_memory/memory_store.rs @@ -110,11 +110,15 @@ where } } -impl TransactableStorage for MemoryStore +impl TransactableStorage for MemoryStore where Description: DatabaseDescription, { - fn commit_changes(&self, changes: Changes) -> StorageResult<()> { + fn commit_changes( + &self, + _: Option, + changes: Changes, + ) -> StorageResult<()> { for (column, btree) in changes.into_iter() { let mut lock = self.inner[column as usize].lock().expect("poisoned"); @@ -156,7 +160,7 @@ mod tests { let mut transaction = self.read_transaction(); let len = transaction.write(key, column, buf)?; let changes = transaction.into_changes(); - self.commit_changes(changes)?; + self.commit_changes(Default::default(), changes)?; Ok(len) } @@ -164,7 +168,7 @@ mod tests { let mut transaction = self.read_transaction(); transaction.delete(key, column)?; let changes = transaction.into_changes(); - self.commit_changes(changes)?; + self.commit_changes(Default::default(), changes)?; Ok(()) } } diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 3a8f5ea5bd9..f010a7eb608 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -502,11 +502,15 @@ where } } -impl TransactableStorage for RocksDb +impl TransactableStorage for RocksDb where Description: DatabaseDescription, { - fn commit_changes(&self, changes: Changes) -> StorageResult<()> { + fn commit_changes( + &self, + _: Option, + changes: Changes, + ) -> StorageResult<()> { let mut batch = WriteBatch::default(); for (column, ops) in changes { @@ -573,7 +577,7 @@ mod tests { let mut transaction = self.read_transaction(); let len = transaction.write(key, column, buf)?; let changes = transaction.into_changes(); - self.commit_changes(changes)?; + self.commit_changes(Default::default(), changes)?; Ok(len) } @@ -582,7 +586,7 @@ mod tests { let mut transaction = self.read_transaction(); transaction.delete(key, column)?; let changes = transaction.into_changes(); - self.commit_changes(changes)?; + self.commit_changes(Default::default(), changes)?; Ok(()) } } @@ -657,7 +661,8 @@ mod tests { )]), )]; - db.commit_changes(HashMap::from_iter(ops)).unwrap(); + db.commit_changes(Default::default(), HashMap::from_iter(ops)) + .unwrap(); assert_eq!(db.get(&key, Column::Metadata).unwrap().unwrap(), value) } @@ -673,7 +678,8 @@ mod tests { Column::Metadata.id(), BTreeMap::from_iter(vec![(key.clone(), WriteOperation::Remove)]), )]; - db.commit_changes(HashMap::from_iter(ops)).unwrap(); + db.commit_changes(Default::default(), HashMap::from_iter(ops)) + .unwrap(); assert_eq!(db.get(&key, Column::Metadata).unwrap(), None); } diff --git a/crates/services/p2p/src/service.rs b/crates/services/p2p/src/service.rs index c6d35ff8a33..943d049b8ad 100644 --- a/crates/services/p2p/src/service.rs +++ b/crates/services/p2p/src/service.rs @@ -895,8 +895,8 @@ pub mod tests { type Height = BlockHeight; - fn latest_height(&self) -> Self::Height { - BlockHeight::default() + fn latest_height(&self) -> Option { + Some(BlockHeight::default()) } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -1024,8 +1024,8 @@ pub mod tests { type Height = BlockHeight; - fn latest_height(&self) -> Self::Height { - BlockHeight::default() + fn latest_height(&self) -> Option { + Some(BlockHeight::default()) } fn view_at(&self, _: &BlockHeight) -> StorageResult { diff --git a/crates/services/producer/src/block_producer.rs b/crates/services/producer/src/block_producer.rs index 24937fef043..68a0ea5bb5e 100644 --- a/crates/services/producer/src/block_producer.rs +++ b/crates/services/producer/src/block_producer.rs @@ -189,6 +189,7 @@ where let height = height.unwrap_or_else(|| { self.view_provider .latest_height() + .unwrap_or_default() .succ() .expect("It is impossible to overflow the current block height") }); diff --git a/crates/services/producer/src/mocks.rs b/crates/services/producer/src/mocks.rs index cf32f2d96af..1409bbabc6b 100644 --- a/crates/services/producer/src/mocks.rs +++ b/crates/services/producer/src/mocks.rs @@ -179,10 +179,10 @@ impl AtomicView for MockDb { type Height = BlockHeight; - fn latest_height(&self) -> BlockHeight { + fn latest_height(&self) -> Option { let blocks = self.blocks.lock().unwrap(); - blocks.keys().max().cloned().unwrap_or_default() + blocks.keys().max().cloned() } fn view_at(&self, _: &BlockHeight) -> StorageResult { diff --git a/crates/services/relayer/src/service/get_logs.rs b/crates/services/relayer/src/service/get_logs.rs index 8ceb079f576..515def95fd9 100644 --- a/crates/services/relayer/src/service/get_logs.rs +++ b/crates/services/relayer/src/service/get_logs.rs @@ -95,6 +95,8 @@ where } } + // TODO: For https://github.com/FuelLabs/fuel-core/issues/451 we need to write each height + // (not only the last height), even if it's empty. if !inserted_last_height { database.insert_events(&last_height, &[])?; } diff --git a/crates/services/relayer/src/storage.rs b/crates/services/relayer/src/storage.rs index 1cdf0628f99..4a08e61259a 100644 --- a/crates/services/relayer/src/storage.rs +++ b/crates/services/relayer/src/storage.rs @@ -85,6 +85,7 @@ impl Mappable for DaHeightTable { /// changed from a unit value. const METADATA_KEY: () = (); +// TODO: Remove `DaHeightTable` and logic associated with it, since the height tracking is controlled by the database impl TableWithBlueprint for DaHeightTable { type Blueprint = Plain>; type Column = Column; diff --git a/crates/services/txpool/src/mock_db.rs b/crates/services/txpool/src/mock_db.rs index 0da650294e5..bbaa5bc15e9 100644 --- a/crates/services/txpool/src/mock_db.rs +++ b/crates/services/txpool/src/mock_db.rs @@ -103,8 +103,8 @@ impl AtomicView for MockDBProvider { type Height = BlockHeight; - fn latest_height(&self) -> Self::Height { - BlockHeight::default() + fn latest_height(&self) -> Option { + Some(BlockHeight::default()) } fn view_at(&self, _: &BlockHeight) -> StorageResult { diff --git a/crates/storage/src/transactional.rs b/crates/storage/src/transactional.rs index b38c3597511..156d8e3eff6 100644 --- a/crates/storage/src/transactional.rs +++ b/crates/storage/src/transactional.rs @@ -41,7 +41,7 @@ pub trait AtomicView: Send + Sync { type Height; /// Returns the latest block height. - fn latest_height(&self) -> Self::Height; + fn latest_height(&self) -> Option; /// Returns the view of the storage at the given `height`. fn view_at(&self, height: &Self::Height) -> StorageResult; diff --git a/tests/tests/blocks.rs b/tests/tests/blocks.rs index b92390cf468..da756fdbea1 100644 --- a/tests/tests/blocks.rs +++ b/tests/tests/blocks.rs @@ -19,6 +19,7 @@ use fuel_core_storage::{ FuelBlocks, SealedBlockConsensus, }, + transactional::WriteTransaction, vm_storage::VmStorageRequirements, StorageAsMut, }; @@ -55,10 +56,16 @@ async fn block() { .unwrap(); let client = FuelClient::from(srv.bound_address); - db.storage::().insert(&height, &block).unwrap(); - db.storage::() + let mut transaction = db.write_transaction(); + transaction + .storage::() + .insert(&height, &block) + .unwrap(); + transaction + .storage::() .insert(&height, &Consensus::PoA(Default::default())) .unwrap(); + transaction.commit().unwrap(); // run test let block = client.block_by_height(height).await.unwrap();