From e87c64d6c3679dfc8aa989254f9d1047c2d8e3a7 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 3 Nov 2023 14:10:57 +0000 Subject: [PATCH] default snapshot filename doesnt have configuration --- bin/reth/src/db/snapshots/headers.rs | 39 ++++++++++++++------- bin/reth/src/db/snapshots/receipts.rs | 41 ++++++++++++++--------- bin/reth/src/db/snapshots/transactions.rs | 40 +++++++++++++--------- crates/primitives/src/snapshot/segment.rs | 26 ++++++-------- crates/snapshot/src/segments/mod.rs | 4 +-- 5 files changed, 89 insertions(+), 61 deletions(-) diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index 6533dd881759..cc3bd0a0b152 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -12,8 +12,11 @@ use reth_primitives::{ use reth_provider::{ providers::SnapshotProvider, DatabaseProviderRO, HeaderProvider, ProviderError, ProviderFactory, }; -use reth_snapshot::segments::{Headers, Segment}; -use std::{path::Path, sync::Arc}; +use reth_snapshot::{segments, segments::Segment}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; impl Command { pub(crate) fn generate_headers_snapshot( @@ -23,15 +26,24 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let segment = Headers::new( - compression, - if self.with_filters { - Filters::WithFilters(inclusion_filter, phf) - } else { - Filters::WithoutFilters - }, - ); - segment.snapshot::(provider, self.from..=(self.from + self.block_interval - 1))?; + let range = self.from..=(self.from + self.block_interval - 1); + let filters = if self.with_filters { + Filters::WithFilters(inclusion_filter, phf) + } else { + Filters::WithoutFilters + }; + + let segment = segments::Headers::new(compression, filters); + + segment.snapshot::(provider, range.clone())?; + + // Default name doesn't have any configuration + let default_name: PathBuf = SnapshotSegment::Headers.filename(&range).into(); + let new_name: PathBuf = SnapshotSegment::Headers + .filename_with_configuration(filters, compression, &range) + .into(); + + std::fs::rename(default_name, new_name)?; Ok(()) } @@ -55,8 +67,9 @@ impl Command { let mut row_indexes = range.clone().collect::>(); let mut rng = rand::thread_rng(); - let path = - SnapshotSegment::Headers.filename_with_configuration(filters, compression, &range); + let path = SnapshotSegment::Headers + .filename_with_configuration(filters, compression, &range) + .into(); let provider = SnapshotProvider::default(); let jar_provider = provider.get_segment_provider(SnapshotSegment::Headers, self.from, Some(path))?; diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index ffe09814e2aa..70a95f31b9b8 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -14,7 +14,10 @@ use reth_provider::{ ReceiptProvider, TransactionsProvider, TransactionsProviderExt, }; use reth_snapshot::{segments, segments::Segment}; -use std::{path::Path, sync::Arc}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; impl Command { pub(crate) fn generate_receipts_snapshot( @@ -24,15 +27,24 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let segment = segments::Receipts::new( - compression, - if self.with_filters { - Filters::WithFilters(inclusion_filter, phf) - } else { - Filters::WithoutFilters - }, - ); - segment.snapshot::(provider, self.from..=(self.from + self.block_interval - 1))?; + let range = self.from..=(self.from + self.block_interval - 1); + let filters = if self.with_filters { + Filters::WithFilters(inclusion_filter, phf) + } else { + Filters::WithoutFilters + }; + + let segment = segments::Receipts::new(compression, filters); + + segment.snapshot::(provider, range.clone())?; + + // Default name doesn't have any configuration + let default_name: PathBuf = SnapshotSegment::Receipts.filename(&range).into(); + let new_name: PathBuf = SnapshotSegment::Receipts + .filename_with_configuration(filters, compression, &range) + .into(); + + std::fs::rename(default_name, new_name)?; Ok(()) } @@ -62,11 +74,10 @@ impl Command { let mut row_indexes = tx_range.clone().collect::>(); - let path = SnapshotSegment::Receipts.filename_with_configuration( - filters, - compression, - &block_range, - ); + let path = SnapshotSegment::Receipts + .filename_with_configuration(filters, compression, &block_range) + .into(); + let provider = SnapshotProvider::default(); let jar_provider = provider.get_segment_provider(SnapshotSegment::Receipts, self.from, Some(path))?; diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index a52c33ddb7f9..a2ced80a3f6a 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -14,7 +14,10 @@ use reth_provider::{ TransactionsProvider, TransactionsProviderExt, }; use reth_snapshot::{segments, segments::Segment}; -use std::{path::Path, sync::Arc}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; impl Command { pub(crate) fn generate_transactions_snapshot( @@ -24,15 +27,24 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let segment = segments::Transactions::new( - compression, - if self.with_filters { - Filters::WithFilters(inclusion_filter, phf) - } else { - Filters::WithoutFilters - }, - ); - segment.snapshot::(provider, self.from..=(self.from + self.block_interval - 1))?; + let range = self.from..=(self.from + self.block_interval - 1); + let filters = if self.with_filters { + Filters::WithFilters(inclusion_filter, phf) + } else { + Filters::WithoutFilters + }; + + let segment = segments::Transactions::new(compression, filters); + + segment.snapshot::(provider, range.clone())?; + + // Default name doesn't have any configuration + let default_name: PathBuf = SnapshotSegment::Transactions.filename(&range).into(); + let new_name: PathBuf = SnapshotSegment::Transactions + .filename_with_configuration(filters, compression, &range) + .into(); + + std::fs::rename(default_name, new_name)?; Ok(()) } @@ -62,11 +74,9 @@ impl Command { let mut row_indexes = tx_range.clone().collect::>(); - let path = SnapshotSegment::Transactions.filename_with_configuration( - filters, - compression, - &block_range, - ); + let path = SnapshotSegment::Transactions + .filename_with_configuration(filters, compression, &block_range) + .into(); let provider = SnapshotProvider::default(); let jar_provider = provider.get_segment_provider(SnapshotSegment::Transactions, self.from, Some(path))?; diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index e59268473cb1..1854f10921b2 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -3,7 +3,7 @@ use crate::{ BlockNumber, TxNumber, }; use serde::{Deserialize, Serialize}; -use std::{ops::RangeInclusive, path::PathBuf, str::FromStr}; +use std::{ops::RangeInclusive, str::FromStr}; use strum::{AsRefStr, EnumString}; #[derive( @@ -36,7 +36,7 @@ pub enum SnapshotSegment { impl SnapshotSegment { /// Returns the default configuration of the segment. - const fn config(&self) -> (Filters, Compression) { + pub const fn config(&self) -> (Filters, Compression) { let default_config = ( Filters::WithFilters(InclusionFilter::Cuckoo, super::PerfectHashingFunction::Fmph), Compression::Lz4, @@ -50,19 +50,20 @@ impl SnapshotSegment { } /// Returns the default file name for the provided segment and range. - pub fn filename(&self, range: &RangeInclusive) -> PathBuf { - let (filters, compression) = self.config(); - self.filename_with_configuration(filters, compression, range) + pub fn filename(&self, range: &RangeInclusive) -> String { + // ATTENTION: if changing the name format, be sure to reflect those changes in + // [`Self::parse_filename`.] + format!("snapshot_{}_{}_{}", self.as_ref(), range.start(), range.end(),) } - /// Returns file name for the provided segment, filters, compression and range. + /// Returns file name for the provided segment and range, alongisde filters, compression. pub fn filename_with_configuration( &self, filters: Filters, compression: Compression, range: &RangeInclusive, - ) -> PathBuf { - let segment_name = self.as_ref(); + ) -> String { + let prefix = self.filename(range); let filters_name = match filters { Filters::WithFilters(inclusion_filter, phf) => { @@ -73,14 +74,7 @@ impl SnapshotSegment { // ATTENTION: if changing the name format, be sure to reflect those changes in // [`Self::parse_filename`.] - format!( - "snapshot_{segment_name}_{}_{}_{}_{}", - range.start(), - range.end(), - filters_name, - compression.as_ref() - ) - .into() + format!("{prefix}_{}_{}", filters_name, compression.as_ref()) } /// Takes a filename and parses the [`SnapshotSegment`] and its inclusive range. diff --git a/crates/snapshot/src/segments/mod.rs b/crates/snapshot/src/segments/mod.rs index 9a8bb462789f..6293c3896c10 100644 --- a/crates/snapshot/src/segments/mod.rs +++ b/crates/snapshot/src/segments/mod.rs @@ -19,7 +19,7 @@ use reth_primitives::{ BlockNumber, SnapshotSegment, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; -use std::ops::RangeInclusive; +use std::{ops::RangeInclusive, path::Path}; pub(crate) type Rows = [Vec>; COLUMNS]; @@ -61,7 +61,7 @@ pub(crate) fn prepare_jar( let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; let mut nippy_jar = NippyJar::new( COLUMNS, - &segment.filename_with_configuration(filters, compression, &block_range), + Path::new(segment.filename(&block_range).as_str()), SegmentHeader::new(block_range, tx_range, segment), );