diff --git a/Cargo.lock b/Cargo.lock index 76615d5719ce..92cbc1734016 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6233,7 +6233,6 @@ dependencies = [ "arbitrary", "assert_matches", "backon", - "boyer-moore-magiclen", "clap", "comfy-table", "confy", @@ -6700,6 +6699,7 @@ name = "reth-db-common" version = "1.0.0" dependencies = [ "alloy-genesis", + "boyer-moore-magiclen", "eyre", "reth-chainspec", "reth-codecs", @@ -6707,6 +6707,7 @@ dependencies = [ "reth-db", "reth-db-api", "reth-etl", + "reth-fs-util", "reth-primitives", "reth-primitives-traits", "reth-provider", diff --git a/Cargo.toml b/Cargo.toml index a7e1ab3d92ae..e0b2cf90b447 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -439,6 +439,7 @@ sha2 = { version = "0.10", default-features = false } paste = "1.0" url = "2.3" backon = "0.4" +boyer-moore-magiclen = "0.2.16" # metrics metrics = "0.23.0" diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index dd7b6ee034a0..8c8e8a0e71a2 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -116,7 +116,6 @@ backon.workspace = true similar-asserts.workspace = true itertools.workspace = true rayon.workspace = true -boyer-moore-magiclen = "0.2.16" ahash = "0.8" # p2p diff --git a/bin/reth/src/commands/db/checksum.rs b/bin/reth/src/commands/db/checksum.rs index 6aa6b69e6d3b..9af9a2321637 100644 --- a/bin/reth/src/commands/db/checksum.rs +++ b/bin/reth/src/commands/db/checksum.rs @@ -1,11 +1,9 @@ -use crate::{ - commands::db::get::{maybe_json_value_parser, table_key}, - utils::DbTool, -}; +use crate::commands::db::get::{maybe_json_value_parser, table_key}; use ahash::RandomState; use clap::Parser; use reth_db::{DatabaseEnv, RawKey, RawTable, RawValue, TableViewer, Tables}; use reth_db_api::{cursor::DbCursorRO, database::Database, table::Table, transaction::DbTx}; +use reth_db_common::DbTool; use std::{ hash::{BuildHasher, Hasher}, sync::Arc, diff --git a/bin/reth/src/commands/db/diff.rs b/bin/reth/src/commands/db/diff.rs index 246b107fa4a1..cd9e24c1d761 100644 --- a/bin/reth/src/commands/db/diff.rs +++ b/bin/reth/src/commands/db/diff.rs @@ -1,11 +1,11 @@ use crate::{ args::DatabaseArgs, dirs::{DataDirPath, PlatformPath}, - utils::DbTool, }; use clap::Parser; use reth_db::{open_db_read_only, tables_to_generic, DatabaseEnv, Tables}; use reth_db_api::{cursor::DbCursorRO, database::Database, table::Table, transaction::DbTx}; +use reth_db_common::DbTool; use std::{ collections::HashMap, fmt::Debug, diff --git a/bin/reth/src/commands/db/get.rs b/bin/reth/src/commands/db/get.rs index a4987e4b4281..cd721a1db4b1 100644 --- a/bin/reth/src/commands/db/get.rs +++ b/bin/reth/src/commands/db/get.rs @@ -1,4 +1,3 @@ -use crate::utils::DbTool; use clap::Parser; use reth_db::{ static_file::{ColumnSelectorOne, ColumnSelectorTwo, HeaderMask, ReceiptMask, TransactionMask}, @@ -8,6 +7,7 @@ use reth_db_api::{ database::Database, table::{Decompress, DupSort, Table}, }; +use reth_db_common::DbTool; use reth_primitives::{BlockHash, Header}; use reth_provider::StaticFileProviderFactory; use reth_static_file_types::StaticFileSegment; diff --git a/bin/reth/src/commands/db/list.rs b/bin/reth/src/commands/db/list.rs index dd1a1846acbd..ed337bdcf81e 100644 --- a/bin/reth/src/commands/db/list.rs +++ b/bin/reth/src/commands/db/list.rs @@ -1,9 +1,9 @@ use super::tui::DbListTUI; -use crate::utils::{DbTool, ListFilter}; use clap::Parser; use eyre::WrapErr; use reth_db::{DatabaseEnv, RawValue, TableViewer, Tables}; use reth_db_api::{database::Database, table::Table}; +use reth_db_common::{DbTool, ListFilter}; use reth_primitives::hex; use std::{cell::RefCell, sync::Arc}; use tracing::error; diff --git a/bin/reth/src/commands/db/mod.rs b/bin/reth/src/commands/db/mod.rs index fcafcc41ac09..736d825c31b2 100644 --- a/bin/reth/src/commands/db/mod.rs +++ b/bin/reth/src/commands/db/mod.rs @@ -1,11 +1,9 @@ //! Database debugging tool -use crate::{ - commands::common::{AccessRights, Environment, EnvironmentArgs}, - utils::DbTool, -}; +use crate::commands::common::{AccessRights, Environment, EnvironmentArgs}; use clap::{Parser, Subcommand}; use reth_db::version::{get_db_version, DatabaseVersionError, DB_VERSION}; +use reth_db_common::DbTool; use std::io::{self, Write}; mod checksum; diff --git a/bin/reth/src/commands/db/stats.rs b/bin/reth/src/commands/db/stats.rs index 9e1eebd95220..b1a979e54918 100644 --- a/bin/reth/src/commands/db/stats.rs +++ b/bin/reth/src/commands/db/stats.rs @@ -1,4 +1,4 @@ -use crate::{commands::db::checksum::ChecksumViewer, utils::DbTool}; +use crate::commands::db::checksum::ChecksumViewer; use clap::Parser; use comfy_table::{Cell, Row, Table as ComfyTable}; use eyre::WrapErr; @@ -6,6 +6,7 @@ use human_bytes::human_bytes; use itertools::Itertools; use reth_db::{mdbx, static_file::iter_static_files, DatabaseEnv, TableViewer, Tables}; use reth_db_api::database::Database; +use reth_db_common::DbTool; use reth_fs_util as fs; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_provider::providers::StaticFileProvider; diff --git a/bin/reth/src/commands/stage/drop.rs b/bin/reth/src/commands/stage/drop.rs index 320c88682a8d..ec32af330e97 100644 --- a/bin/reth/src/commands/stage/drop.rs +++ b/bin/reth/src/commands/stage/drop.rs @@ -3,13 +3,15 @@ use crate::{ args::StageEnum, commands::common::{AccessRights, Environment, EnvironmentArgs}, - utils::DbTool, }; use clap::Parser; use itertools::Itertools; use reth_db::{static_file::iter_static_files, tables, DatabaseEnv}; use reth_db_api::transaction::DbTxMut; -use reth_db_common::init::{insert_genesis_header, insert_genesis_history, insert_genesis_state}; +use reth_db_common::{ + init::{insert_genesis_header, insert_genesis_history, insert_genesis_state}, + DbTool, +}; use reth_provider::{providers::StaticFileWriter, StaticFileProviderFactory}; use reth_stages::StageId; use reth_static_file_types::{find_fixed_range, StaticFileSegment}; diff --git a/bin/reth/src/commands/stage/dump/execution.rs b/bin/reth/src/commands/stage/dump/execution.rs index b6d6721dcf8d..67b6d5a659c4 100644 --- a/bin/reth/src/commands/stage/dump/execution.rs +++ b/bin/reth/src/commands/stage/dump/execution.rs @@ -1,9 +1,10 @@ use super::setup; -use crate::{macros::block_executor, utils::DbTool}; +use crate::macros::block_executor; use reth_db::{tables, DatabaseEnv}; use reth_db_api::{ cursor::DbCursorRO, database::Database, table::TableImporter, transaction::DbTx, }; +use reth_db_common::DbTool; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_provider::{providers::StaticFileProvider, ChainSpecProvider, ProviderFactory}; use reth_stages::{stages::ExecutionStage, Stage, StageCheckpoint, UnwindInput}; diff --git a/bin/reth/src/commands/stage/dump/hashing_account.rs b/bin/reth/src/commands/stage/dump/hashing_account.rs index 2e50a8ad6059..899b521fdc57 100644 --- a/bin/reth/src/commands/stage/dump/hashing_account.rs +++ b/bin/reth/src/commands/stage/dump/hashing_account.rs @@ -1,8 +1,8 @@ use super::setup; -use crate::utils::DbTool; use eyre::Result; use reth_db::{tables, DatabaseEnv}; use reth_db_api::{database::Database, table::TableImporter}; +use reth_db_common::DbTool; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_primitives::BlockNumber; use reth_provider::{providers::StaticFileProvider, ProviderFactory}; diff --git a/bin/reth/src/commands/stage/dump/hashing_storage.rs b/bin/reth/src/commands/stage/dump/hashing_storage.rs index 1dfd722f5099..f05ac390dc8e 100644 --- a/bin/reth/src/commands/stage/dump/hashing_storage.rs +++ b/bin/reth/src/commands/stage/dump/hashing_storage.rs @@ -1,8 +1,8 @@ use super::setup; -use crate::utils::DbTool; use eyre::Result; use reth_db::{tables, DatabaseEnv}; use reth_db_api::{database::Database, table::TableImporter}; +use reth_db_common::DbTool; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_provider::{providers::StaticFileProvider, ProviderFactory}; use reth_stages::{stages::StorageHashingStage, Stage, StageCheckpoint, UnwindInput}; diff --git a/bin/reth/src/commands/stage/dump/merkle.rs b/bin/reth/src/commands/stage/dump/merkle.rs index fa345bb474a4..a81d5f524539 100644 --- a/bin/reth/src/commands/stage/dump/merkle.rs +++ b/bin/reth/src/commands/stage/dump/merkle.rs @@ -1,9 +1,10 @@ use super::setup; -use crate::{macros::block_executor, utils::DbTool}; +use crate::macros::block_executor; use eyre::Result; use reth_config::config::EtlConfig; use reth_db::{tables, DatabaseEnv}; use reth_db_api::{database::Database, table::TableImporter}; +use reth_db_common::DbTool; use reth_exex::ExExManagerHandle; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_primitives::BlockNumber; diff --git a/bin/reth/src/commands/stage/dump/mod.rs b/bin/reth/src/commands/stage/dump/mod.rs index 287708b00d68..f1fbdbbdecbd 100644 --- a/bin/reth/src/commands/stage/dump/mod.rs +++ b/bin/reth/src/commands/stage/dump/mod.rs @@ -1,18 +1,17 @@ //! Database debugging tool use crate::{ + args::DatadirArgs, commands::common::{AccessRights, Environment, EnvironmentArgs}, dirs::DataDirPath, - utils::DbTool, }; - -use crate::args::DatadirArgs; use clap::Parser; use reth_db::{init_db, mdbx::DatabaseArguments, tables, DatabaseEnv}; use reth_db_api::{ cursor::DbCursorRO, database::Database, models::ClientVersion, table::TableImporter, transaction::DbTx, }; +use reth_db_common::DbTool; use reth_node_core::dirs::PlatformPath; use std::path::PathBuf; use tracing::info; diff --git a/bin/reth/src/utils.rs b/bin/reth/src/utils.rs index 1dd4f6893c1f..cf8985b290d9 100644 --- a/bin/reth/src/utils.rs +++ b/bin/reth/src/utils.rs @@ -1,21 +1,5 @@ //! Common CLI utility functions. -use boyer_moore_magiclen::BMByte; -use eyre::Result; -use reth_chainspec::ChainSpec; -use reth_db::{RawTable, TableRawRow}; -use reth_db_api::{ - cursor::{DbCursorRO, DbDupCursorRO}, - database::Database, - table::{Decode, Decompress, DupSort, Table, TableRow}, - transaction::{DbTx, DbTxMut}, - DatabaseError, -}; -use reth_fs_util as fs; -use reth_provider::{ChainSpecProvider, ProviderFactory}; -use std::{path::Path, rc::Rc, sync::Arc}; -use tracing::info; - /// Exposing `open_db_read_only` function pub mod db { pub use reth_db::open_db_read_only; @@ -24,175 +8,3 @@ pub mod db { /// Re-exported from `reth_node_core`, also to prevent a breaking change. See the comment on /// the `reth_node_core::args` re-export for more details. pub use reth_node_core::utils::*; - -/// Wrapper over DB that implements many useful DB queries. -#[derive(Debug)] -pub struct DbTool { - /// The provider factory that the db tool will use. - pub provider_factory: ProviderFactory, -} - -impl DbTool { - /// Takes a DB where the tables have already been created. - pub fn new(provider_factory: ProviderFactory) -> eyre::Result { - // Disable timeout because we are entering a TUI which might read for a long time. We - // disable on the [`DbTool`] level since it's only used in the CLI. - provider_factory.provider()?.disable_long_read_transaction_safety(); - Ok(Self { provider_factory }) - } - - /// Get an [`Arc`] to the [`ChainSpec`]. - pub fn chain(&self) -> Arc { - self.provider_factory.chain_spec() - } - - /// Grabs the contents of the table within a certain index range and places the - /// entries into a [`HashMap`][std::collections::HashMap]. - /// - /// [`ListFilter`] can be used to further - /// filter down the desired results. (eg. List only rows which include `0xd3adbeef`) - pub fn list(&self, filter: &ListFilter) -> Result<(Vec>, usize)> { - let bmb = Rc::new(BMByte::from(&filter.search)); - if bmb.is_none() && filter.has_search() { - eyre::bail!("Invalid search.") - } - - let mut hits = 0; - - let data = self.provider_factory.db_ref().view(|tx| { - let mut cursor = - tx.cursor_read::>().expect("Was not able to obtain a cursor."); - - let map_filter = |row: Result, _>| { - if let Ok((k, v)) = row { - let (key, value) = (k.into_key(), v.into_value()); - - if key.len() + value.len() < filter.min_row_size { - return None - } - if key.len() < filter.min_key_size { - return None - } - if value.len() < filter.min_value_size { - return None - } - - let result = || { - if filter.only_count { - return None - } - Some(( - ::Key::decode(&key).unwrap(), - ::Value::decompress(&value).unwrap(), - )) - }; - - match &*bmb { - Some(searcher) => { - if searcher.find_first_in(&value).is_some() || - searcher.find_first_in(&key).is_some() - { - hits += 1; - return result() - } - } - None => { - hits += 1; - return result() - } - } - } - None - }; - - if filter.reverse { - Ok(cursor - .walk_back(None)? - .skip(filter.skip) - .filter_map(map_filter) - .take(filter.len) - .collect::>()) - } else { - Ok(cursor - .walk(None)? - .skip(filter.skip) - .filter_map(map_filter) - .take(filter.len) - .collect::>()) - } - })?; - - Ok((data.map_err(|e: DatabaseError| eyre::eyre!(e))?, hits)) - } - - /// Grabs the content of the table for the given key - pub fn get(&self, key: T::Key) -> Result> { - self.provider_factory.db_ref().view(|tx| tx.get::(key))?.map_err(|e| eyre::eyre!(e)) - } - - /// Grabs the content of the `DupSort` table for the given key and subkey - pub fn get_dup(&self, key: T::Key, subkey: T::SubKey) -> Result> { - self.provider_factory - .db_ref() - .view(|tx| tx.cursor_dup_read::()?.seek_by_key_subkey(key, subkey))? - .map_err(|e| eyre::eyre!(e)) - } - - /// Drops the database and the static files at the given path. - pub fn drop( - &self, - db_path: impl AsRef, - static_files_path: impl AsRef, - ) -> Result<()> { - let db_path = db_path.as_ref(); - info!(target: "reth::cli", "Dropping database at {:?}", db_path); - fs::remove_dir_all(db_path)?; - - let static_files_path = static_files_path.as_ref(); - info!(target: "reth::cli", "Dropping static files at {:?}", static_files_path); - fs::remove_dir_all(static_files_path)?; - fs::create_dir_all(static_files_path)?; - - Ok(()) - } - - /// Drops the provided table from the database. - pub fn drop_table(&self) -> Result<()> { - self.provider_factory.db_ref().update(|tx| tx.clear::())??; - Ok(()) - } -} - -/// Filters the results coming from the database. -#[derive(Debug)] -pub struct ListFilter { - /// Skip first N entries. - pub skip: usize, - /// Take N entries. - pub len: usize, - /// Sequence of bytes that will be searched on values and keys from the database. - pub search: Vec, - /// Minimum row size. - pub min_row_size: usize, - /// Minimum key size. - pub min_key_size: usize, - /// Minimum value size. - pub min_value_size: usize, - /// Reverse order of entries. - pub reverse: bool, - /// Only counts the number of filtered entries without decoding and returning them. - pub only_count: bool, -} - -impl ListFilter { - /// If `search` has a list of bytes, then filter for rows that have this sequence. - pub fn has_search(&self) -> bool { - !self.search.is_empty() - } - - /// Updates the page with new `skip` and `len` values. - pub fn update_page(&mut self, skip: usize, len: usize) { - self.skip = skip; - self.len = len; - } -} diff --git a/crates/storage/db-common/Cargo.toml b/crates/storage/db-common/Cargo.toml index c2760f672793..d80236defd32 100644 --- a/crates/storage/db-common/Cargo.toml +++ b/crates/storage/db-common/Cargo.toml @@ -19,6 +19,7 @@ reth-trie.workspace = true reth-etl.workspace = true reth-codecs.workspace = true reth-stages-types.workspace = true +reth-fs-util.workspace = true # eth alloy-genesis.workspace = true @@ -26,6 +27,7 @@ alloy-genesis.workspace = true # misc eyre.workspace = true thiserror.workspace = true +boyer-moore-magiclen.workspace = true # io serde.workspace = true diff --git a/crates/storage/db-common/src/db_tool/mod.rs b/crates/storage/db-common/src/db_tool/mod.rs new file mode 100644 index 000000000000..3884089b4370 --- /dev/null +++ b/crates/storage/db-common/src/db_tool/mod.rs @@ -0,0 +1,189 @@ +//! Common db operations + +use boyer_moore_magiclen::BMByte; +use eyre::Result; +use reth_chainspec::ChainSpec; +use reth_db::{RawTable, TableRawRow}; +use reth_db_api::{ + cursor::{DbCursorRO, DbDupCursorRO}, + database::Database, + table::{Decode, Decompress, DupSort, Table, TableRow}, + transaction::{DbTx, DbTxMut}, + DatabaseError, +}; +use reth_fs_util as fs; +use reth_provider::{ChainSpecProvider, ProviderFactory}; +use std::{path::Path, rc::Rc, sync::Arc}; +use tracing::info; + +/// Wrapper over DB that implements many useful DB queries. +#[derive(Debug)] +pub struct DbTool { + /// The provider factory that the db tool will use. + pub provider_factory: ProviderFactory, +} + +impl DbTool { + /// Takes a DB where the tables have already been created. + pub fn new(provider_factory: ProviderFactory) -> eyre::Result { + // Disable timeout because we are entering a TUI which might read for a long time. We + // disable on the [`DbTool`] level since it's only used in the CLI. + provider_factory.provider()?.disable_long_read_transaction_safety(); + Ok(Self { provider_factory }) + } + + /// Get an [`Arc`] to the [`ChainSpec`]. + pub fn chain(&self) -> Arc { + self.provider_factory.chain_spec() + } + + /// Grabs the contents of the table within a certain index range and places the + /// entries into a [`HashMap`][std::collections::HashMap]. + /// + /// [`ListFilter`] can be used to further + /// filter down the desired results. (eg. List only rows which include `0xd3adbeef`) + pub fn list(&self, filter: &ListFilter) -> Result<(Vec>, usize)> { + let bmb = Rc::new(BMByte::from(&filter.search)); + if bmb.is_none() && filter.has_search() { + eyre::bail!("Invalid search.") + } + + let mut hits = 0; + + let data = self.provider_factory.db_ref().view(|tx| { + let mut cursor = + tx.cursor_read::>().expect("Was not able to obtain a cursor."); + + let map_filter = |row: Result, _>| { + if let Ok((k, v)) = row { + let (key, value) = (k.into_key(), v.into_value()); + + if key.len() + value.len() < filter.min_row_size { + return None + } + if key.len() < filter.min_key_size { + return None + } + if value.len() < filter.min_value_size { + return None + } + + let result = || { + if filter.only_count { + return None + } + Some(( + ::Key::decode(&key).unwrap(), + ::Value::decompress(&value).unwrap(), + )) + }; + + match &*bmb { + Some(searcher) => { + if searcher.find_first_in(&value).is_some() || + searcher.find_first_in(&key).is_some() + { + hits += 1; + return result() + } + } + None => { + hits += 1; + return result() + } + } + } + None + }; + + if filter.reverse { + Ok(cursor + .walk_back(None)? + .skip(filter.skip) + .filter_map(map_filter) + .take(filter.len) + .collect::>()) + } else { + Ok(cursor + .walk(None)? + .skip(filter.skip) + .filter_map(map_filter) + .take(filter.len) + .collect::>()) + } + })?; + + Ok((data.map_err(|e: DatabaseError| eyre::eyre!(e))?, hits)) + } + + /// Grabs the content of the table for the given key + pub fn get(&self, key: T::Key) -> Result> { + self.provider_factory.db_ref().view(|tx| tx.get::(key))?.map_err(|e| eyre::eyre!(e)) + } + + /// Grabs the content of the `DupSort` table for the given key and subkey + pub fn get_dup(&self, key: T::Key, subkey: T::SubKey) -> Result> { + self.provider_factory + .db_ref() + .view(|tx| tx.cursor_dup_read::()?.seek_by_key_subkey(key, subkey))? + .map_err(|e| eyre::eyre!(e)) + } + + /// Drops the database and the static files at the given path. + pub fn drop( + &self, + db_path: impl AsRef, + static_files_path: impl AsRef, + ) -> Result<()> { + let db_path = db_path.as_ref(); + info!(target: "reth::cli", "Dropping database at {:?}", db_path); + fs::remove_dir_all(db_path)?; + + let static_files_path = static_files_path.as_ref(); + info!(target: "reth::cli", "Dropping static files at {:?}", static_files_path); + fs::remove_dir_all(static_files_path)?; + fs::create_dir_all(static_files_path)?; + + Ok(()) + } + + /// Drops the provided table from the database. + pub fn drop_table(&self) -> Result<()> { + self.provider_factory.db_ref().update(|tx| tx.clear::())??; + Ok(()) + } +} + +/// Filters the results coming from the database. +#[derive(Debug)] +pub struct ListFilter { + /// Skip first N entries. + pub skip: usize, + /// Take N entries. + pub len: usize, + /// Sequence of bytes that will be searched on values and keys from the database. + pub search: Vec, + /// Minimum row size. + pub min_row_size: usize, + /// Minimum key size. + pub min_key_size: usize, + /// Minimum value size. + pub min_value_size: usize, + /// Reverse order of entries. + pub reverse: bool, + /// Only counts the number of filtered entries without decoding and returning them. + pub only_count: bool, +} + +impl ListFilter { + /// If `search` has a list of bytes, then filter for rows that have this sequence. + pub fn has_search(&self) -> bool { + !self.search.is_empty() + } + + /// Updates the page with new `skip` and `len` values. + pub fn update_page(&mut self, skip: usize, len: usize) { + self.skip = skip; + self.len = len; + } +} diff --git a/crates/storage/db-common/src/lib.rs b/crates/storage/db-common/src/lib.rs index abcbc62762a4..173e53143408 100644 --- a/crates/storage/db-common/src/lib.rs +++ b/crates/storage/db-common/src/lib.rs @@ -9,3 +9,6 @@ #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] pub mod init; + +mod db_tool; +pub use db_tool::*;