refactor: move DbTool type to db-common (#9119)

Cargo.toml

@@ -439,6 +439,7 @@ sha2 = { version = "0.10", default-features = false }
 paste = "1.0"
 url = "2.3"
 backon = "0.4"
+boyer-moore-magiclen = "0.2.16"
 
 # metrics
 metrics = "0.23.0"

bin/reth/Cargo.toml

@@ -116,7 +116,6 @@ backon.workspace = true
 similar-asserts.workspace = true
 itertools.workspace = true
 rayon.workspace = true
-boyer-moore-magiclen = "0.2.16"
 ahash = "0.8"
 
 # p2p

bin/reth/src/commands/db/checksum.rs

@@ -1,11 +1,9 @@
-use crate::{
-    commands::db::get::{maybe_json_value_parser, table_key},
-    utils::DbTool,
-};
+use crate::commands::db::get::{maybe_json_value_parser, table_key};
 use ahash::RandomState;
 use clap::Parser;
 use reth_db::{DatabaseEnv, RawKey, RawTable, RawValue, TableViewer, Tables};
 use reth_db_api::{cursor::DbCursorRO, database::Database, table::Table, transaction::DbTx};
+use reth_db_common::DbTool;
 use std::{
     hash::{BuildHasher, Hasher},
     sync::Arc,

bin/reth/src/commands/db/diff.rs

@@ -1,11 +1,11 @@
 use crate::{
     args::DatabaseArgs,
     dirs::{DataDirPath, PlatformPath},
-    utils::DbTool,
 };
 use clap::Parser;
 use reth_db::{open_db_read_only, tables_to_generic, DatabaseEnv, Tables};
 use reth_db_api::{cursor::DbCursorRO, database::Database, table::Table, transaction::DbTx};
+use reth_db_common::DbTool;
 use std::{
     collections::HashMap,
     fmt::Debug,

bin/reth/src/commands/db/get.rs

@@ -1,4 +1,3 @@
-use crate::utils::DbTool;
 use clap::Parser;
 use reth_db::{
     static_file::{ColumnSelectorOne, ColumnSelectorTwo, HeaderMask, ReceiptMask, TransactionMask},
@@ -8,6 +7,7 @@ use reth_db_api::{
     database::Database,
     table::{Decompress, DupSort, Table},
 };
+use reth_db_common::DbTool;
 use reth_primitives::{BlockHash, Header};
 use reth_provider::StaticFileProviderFactory;
 use reth_static_file_types::StaticFileSegment;

bin/reth/src/commands/db/list.rs

@@ -1,9 +1,9 @@
 use super::tui::DbListTUI;
-use crate::utils::{DbTool, ListFilter};
 use clap::Parser;
 use eyre::WrapErr;
 use reth_db::{DatabaseEnv, RawValue, TableViewer, Tables};
 use reth_db_api::{database::Database, table::Table};
+use reth_db_common::{DbTool, ListFilter};
 use reth_primitives::hex;
 use std::{cell::RefCell, sync::Arc};
 use tracing::error;

bin/reth/src/commands/db/mod.rs

@@ -1,11 +1,9 @@
 //! Database debugging tool
 
-use crate::{
-    commands::common::{AccessRights, Environment, EnvironmentArgs},
-    utils::DbTool,
-};
+use crate::commands::common::{AccessRights, Environment, EnvironmentArgs};
 use clap::{Parser, Subcommand};
 use reth_db::version::{get_db_version, DatabaseVersionError, DB_VERSION};
+use reth_db_common::DbTool;
 use std::io::{self, Write};
 
 mod checksum;

bin/reth/src/commands/db/stats.rs

@@ -1,11 +1,12 @@
-use crate::{commands::db::checksum::ChecksumViewer, utils::DbTool};
+use crate::commands::db::checksum::ChecksumViewer;
 use clap::Parser;
 use comfy_table::{Cell, Row, Table as ComfyTable};
 use eyre::WrapErr;
 use human_bytes::human_bytes;
 use itertools::Itertools;
 use reth_db::{mdbx, static_file::iter_static_files, DatabaseEnv, TableViewer, Tables};
 use reth_db_api::database::Database;
+use reth_db_common::DbTool;
 use reth_fs_util as fs;
 use reth_node_core::dirs::{ChainPath, DataDirPath};
 use reth_provider::providers::StaticFileProvider;

bin/reth/src/commands/stage/drop.rs

@@ -3,13 +3,15 @@
 use crate::{
     args::StageEnum,
     commands::common::{AccessRights, Environment, EnvironmentArgs},
-    utils::DbTool,
 };
 use clap::Parser;
 use itertools::Itertools;
 use reth_db::{static_file::iter_static_files, tables, DatabaseEnv};
 use reth_db_api::transaction::DbTxMut;
-use reth_db_common::init::{insert_genesis_header, insert_genesis_history, insert_genesis_state};
+use reth_db_common::{
+    init::{insert_genesis_header, insert_genesis_history, insert_genesis_state},
+    DbTool,
+};
 use reth_provider::{providers::StaticFileWriter, StaticFileProviderFactory};
 use reth_stages::StageId;
 use reth_static_file_types::{find_fixed_range, StaticFileSegment};

bin/reth/src/commands/stage/dump/execution.rs

@@ -1,9 +1,10 @@
 use super::setup;
-use crate::{macros::block_executor, utils::DbTool};
+use crate::macros::block_executor;
 use reth_db::{tables, DatabaseEnv};
 use reth_db_api::{
     cursor::DbCursorRO, database::Database, table::TableImporter, transaction::DbTx,
 };
+use reth_db_common::DbTool;
 use reth_node_core::dirs::{ChainPath, DataDirPath};
 use reth_provider::{providers::StaticFileProvider, ChainSpecProvider, ProviderFactory};
 use reth_stages::{stages::ExecutionStage, Stage, StageCheckpoint, UnwindInput};

bin/reth/src/commands/stage/dump/hashing_account.rs

@@ -1,8 +1,8 @@
 use super::setup;
-use crate::utils::DbTool;
 use eyre::Result;
 use reth_db::{tables, DatabaseEnv};
 use reth_db_api::{database::Database, table::TableImporter};
+use reth_db_common::DbTool;
 use reth_node_core::dirs::{ChainPath, DataDirPath};
 use reth_primitives::BlockNumber;
 use reth_provider::{providers::StaticFileProvider, ProviderFactory};

bin/reth/src/commands/stage/dump/hashing_storage.rs

@@ -1,8 +1,8 @@
 use super::setup;
-use crate::utils::DbTool;
 use eyre::Result;
 use reth_db::{tables, DatabaseEnv};
 use reth_db_api::{database::Database, table::TableImporter};
+use reth_db_common::DbTool;
 use reth_node_core::dirs::{ChainPath, DataDirPath};
 use reth_provider::{providers::StaticFileProvider, ProviderFactory};
 use reth_stages::{stages::StorageHashingStage, Stage, StageCheckpoint, UnwindInput};

bin/reth/src/commands/stage/dump/merkle.rs

@@ -1,9 +1,10 @@
 use super::setup;
-use crate::{macros::block_executor, utils::DbTool};
+use crate::macros::block_executor;
 use eyre::Result;
 use reth_config::config::EtlConfig;
 use reth_db::{tables, DatabaseEnv};
 use reth_db_api::{database::Database, table::TableImporter};
+use reth_db_common::DbTool;
 use reth_exex::ExExManagerHandle;
 use reth_node_core::dirs::{ChainPath, DataDirPath};
 use reth_primitives::BlockNumber;

bin/reth/src/commands/stage/dump/mod.rs

@@ -1,18 +1,17 @@
 //! Database debugging tool
 
 use crate::{
+    args::DatadirArgs,
     commands::common::{AccessRights, Environment, EnvironmentArgs},
     dirs::DataDirPath,
-    utils::DbTool,
 };
-
-use crate::args::DatadirArgs;
 use clap::Parser;
 use reth_db::{init_db, mdbx::DatabaseArguments, tables, DatabaseEnv};
 use reth_db_api::{
     cursor::DbCursorRO, database::Database, models::ClientVersion, table::TableImporter,
     transaction::DbTx,
 };
+use reth_db_common::DbTool;
 use reth_node_core::dirs::PlatformPath;
 use std::path::PathBuf;
 use tracing::info;

bin/reth/src/utils.rs

@@ -1,21 +1,5 @@
 //! Common CLI utility functions.
 
-use boyer_moore_magiclen::BMByte;
-use eyre::Result;
-use reth_chainspec::ChainSpec;
-use reth_db::{RawTable, TableRawRow};
-use reth_db_api::{
-    cursor::{DbCursorRO, DbDupCursorRO},
-    database::Database,
-    table::{Decode, Decompress, DupSort, Table, TableRow},
-    transaction::{DbTx, DbTxMut},
-    DatabaseError,
-};
-use reth_fs_util as fs;
-use reth_provider::{ChainSpecProvider, ProviderFactory};
-use std::{path::Path, rc::Rc, sync::Arc};
-use tracing::info;
-
 /// Exposing `open_db_read_only` function
 pub mod db {
     pub use reth_db::open_db_read_only;
@@ -24,175 +8,3 @@ pub mod db {
 /// Re-exported from `reth_node_core`, also to prevent a breaking change. See the comment on
 /// the `reth_node_core::args` re-export for more details.
 pub use reth_node_core::utils::*;
-
-/// Wrapper over DB that implements many useful DB queries.
-#[derive(Debug)]
-pub struct DbTool<DB: Database> {
-    /// The provider factory that the db tool will use.
-    pub provider_factory: ProviderFactory<DB>,
-}
-
-impl<DB: Database> DbTool<DB> {
-    /// Takes a DB where the tables have already been created.
-    pub fn new(provider_factory: ProviderFactory<DB>) -> eyre::Result<Self> {
-        // Disable timeout because we are entering a TUI which might read for a long time. We
-        // disable on the [`DbTool`] level since it's only used in the CLI.
-        provider_factory.provider()?.disable_long_read_transaction_safety();
-        Ok(Self { provider_factory })
-    }
-
-    /// Get an [`Arc`] to the [`ChainSpec`].
-    pub fn chain(&self) -> Arc<ChainSpec> {
-        self.provider_factory.chain_spec()
-    }
-
-    /// Grabs the contents of the table within a certain index range and places the
-    /// entries into a [`HashMap`][std::collections::HashMap].
-    ///
-    /// [`ListFilter`] can be used to further
-    /// filter down the desired results. (eg. List only rows which include `0xd3adbeef`)
-    pub fn list<T: Table>(&self, filter: &ListFilter) -> Result<(Vec<TableRow<T>>, usize)> {
-        let bmb = Rc::new(BMByte::from(&filter.search));
-        if bmb.is_none() && filter.has_search() {
-            eyre::bail!("Invalid search.")
-        }
-
-        let mut hits = 0;
-
-        let data = self.provider_factory.db_ref().view(|tx| {
-            let mut cursor =
-                tx.cursor_read::<RawTable<T>>().expect("Was not able to obtain a cursor.");
-
-            let map_filter = |row: Result<TableRawRow<T>, _>| {
-                if let Ok((k, v)) = row {
-                    let (key, value) = (k.into_key(), v.into_value());
-
-                    if key.len() + value.len() < filter.min_row_size {
-                        return None
-                    }
-                    if key.len() < filter.min_key_size {
-                        return None
-                    }
-                    if value.len() < filter.min_value_size {
-                        return None
-                    }
-
-                    let result = || {
-                        if filter.only_count {
-                            return None
-                        }
-                        Some((
-                            <T as Table>::Key::decode(&key).unwrap(),
-                            <T as Table>::Value::decompress(&value).unwrap(),
-                        ))
-                    };
-
-                    match &*bmb {
-                        Some(searcher) => {
-                            if searcher.find_first_in(&value).is_some() ||
-                                searcher.find_first_in(&key).is_some()
-                            {
-                                hits += 1;
-                                return result()
-                            }
-                        }
-                        None => {
-                            hits += 1;
-                            return result()
-                        }
-                    }
-                }
-                None
-            };
-
-            if filter.reverse {
-                Ok(cursor
-                    .walk_back(None)?
-                    .skip(filter.skip)
-                    .filter_map(map_filter)
-                    .take(filter.len)
-                    .collect::<Vec<(_, _)>>())
-            } else {
-                Ok(cursor
-                    .walk(None)?
-                    .skip(filter.skip)
-                    .filter_map(map_filter)
-                    .take(filter.len)
-                    .collect::<Vec<(_, _)>>())
-            }
-        })?;
-
-        Ok((data.map_err(|e: DatabaseError| eyre::eyre!(e))?, hits))
-    }
-
-    /// Grabs the content of the table for the given key
-    pub fn get<T: Table>(&self, key: T::Key) -> Result<Option<T::Value>> {
-        self.provider_factory.db_ref().view(|tx| tx.get::<T>(key))?.map_err(|e| eyre::eyre!(e))
-    }
-
-    /// Grabs the content of the `DupSort` table for the given key and subkey
-    pub fn get_dup<T: DupSort>(&self, key: T::Key, subkey: T::SubKey) -> Result<Option<T::Value>> {
-        self.provider_factory
-            .db_ref()
-            .view(|tx| tx.cursor_dup_read::<T>()?.seek_by_key_subkey(key, subkey))?
-            .map_err(|e| eyre::eyre!(e))
-    }
-
-    /// Drops the database and the static files at the given path.
-    pub fn drop(
-        &self,
-        db_path: impl AsRef<Path>,
-        static_files_path: impl AsRef<Path>,
-    ) -> Result<()> {
-        let db_path = db_path.as_ref();
-        info!(target: "reth::cli", "Dropping database at {:?}", db_path);
-        fs::remove_dir_all(db_path)?;
-
-        let static_files_path = static_files_path.as_ref();
-        info!(target: "reth::cli", "Dropping static files at {:?}", static_files_path);
-        fs::remove_dir_all(static_files_path)?;
-        fs::create_dir_all(static_files_path)?;
-
-        Ok(())
-    }
-
-    /// Drops the provided table from the database.
-    pub fn drop_table<T: Table>(&self) -> Result<()> {
-        self.provider_factory.db_ref().update(|tx| tx.clear::<T>())??;
-        Ok(())
-    }
-}
-
-/// Filters the results coming from the database.
-#[derive(Debug)]
-pub struct ListFilter {
-    /// Skip first N entries.
-    pub skip: usize,
-    /// Take N entries.
-    pub len: usize,
-    /// Sequence of bytes that will be searched on values and keys from the database.
-    pub search: Vec<u8>,
-    /// Minimum row size.
-    pub min_row_size: usize,
-    /// Minimum key size.
-    pub min_key_size: usize,
-    /// Minimum value size.
-    pub min_value_size: usize,
-    /// Reverse order of entries.
-    pub reverse: bool,
-    /// Only counts the number of filtered entries without decoding and returning them.
-    pub only_count: bool,
-}
-
-impl ListFilter {
-    /// If `search` has a list of bytes, then filter for rows that have this sequence.
-    pub fn has_search(&self) -> bool {
-        !self.search.is_empty()
-    }
-
-    /// Updates the page with new `skip` and `len` values.
-    pub fn update_page(&mut self, skip: usize, len: usize) {
-        self.skip = skip;
-        self.len = len;
-    }
-}