Skip to content

Commit

Permalink
default snapshot filename doesnt have configuration
Browse files Browse the repository at this point in the history
  • Loading branch information
@joshieDo
joshieDo committed Nov 3, 2023
1 parent ec89704 commit e87c64d
Show file tree
Hide file tree
Showing 5 changed files with 89 additions and 61 deletions.
39 changes: 26 additions & 13 deletions bin/reth/src/db/snapshots/headers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,11 @@ use reth_primitives::{
use reth_provider::{
providers::SnapshotProvider, DatabaseProviderRO, HeaderProvider, ProviderError, ProviderFactory,
};
use reth_snapshot::segments::{Headers, Segment};
use std::{path::Path, sync::Arc};
use reth_snapshot::{segments, segments::Segment};
use std::{
path::{Path, PathBuf},
sync::Arc,
};

impl Command {
pub(crate) fn generate_headers_snapshot<DB: Database>(
Expand All @@ -23,15 +26,24 @@ impl Command {
inclusion_filter: InclusionFilter,
phf: PerfectHashingFunction,
) -> eyre::Result<()> {
let segment = Headers::new(
compression,
if self.with_filters {
Filters::WithFilters(inclusion_filter, phf)
} else {
Filters::WithoutFilters
},
);
segment.snapshot::<DB>(provider, self.from..=(self.from + self.block_interval - 1))?;
let range = self.from..=(self.from + self.block_interval - 1);
let filters = if self.with_filters {
Filters::WithFilters(inclusion_filter, phf)
} else {
Filters::WithoutFilters
};

let segment = segments::Headers::new(compression, filters);

segment.snapshot::<DB>(provider, range.clone())?;

// Default name doesn't have any configuration
let default_name: PathBuf = SnapshotSegment::Headers.filename(&range).into();
let new_name: PathBuf = SnapshotSegment::Headers
.filename_with_configuration(filters, compression, &range)
.into();

std::fs::rename(default_name, new_name)?;

Ok(())
}
Expand All @@ -55,8 +67,9 @@ impl Command {

let mut row_indexes = range.clone().collect::<Vec<_>>();
let mut rng = rand::thread_rng();
let path =
SnapshotSegment::Headers.filename_with_configuration(filters, compression, &range);
let path = SnapshotSegment::Headers
.filename_with_configuration(filters, compression, &range)
.into();
let provider = SnapshotProvider::default();
let jar_provider =
provider.get_segment_provider(SnapshotSegment::Headers, self.from, Some(path))?;
Expand Down
41 changes: 26 additions & 15 deletions bin/reth/src/db/snapshots/receipts.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@ use reth_provider::{
ReceiptProvider, TransactionsProvider, TransactionsProviderExt,
};
use reth_snapshot::{segments, segments::Segment};
use std::{path::Path, sync::Arc};
use std::{
path::{Path, PathBuf},
sync::Arc,
};

impl Command {
pub(crate) fn generate_receipts_snapshot<DB: Database>(
Expand All @@ -24,15 +27,24 @@ impl Command {
inclusion_filter: InclusionFilter,
phf: PerfectHashingFunction,
) -> eyre::Result<()> {
let segment = segments::Receipts::new(
compression,
if self.with_filters {
Filters::WithFilters(inclusion_filter, phf)
} else {
Filters::WithoutFilters
},
);
segment.snapshot::<DB>(provider, self.from..=(self.from + self.block_interval - 1))?;
let range = self.from..=(self.from + self.block_interval - 1);
let filters = if self.with_filters {
Filters::WithFilters(inclusion_filter, phf)
} else {
Filters::WithoutFilters
};

let segment = segments::Receipts::new(compression, filters);

segment.snapshot::<DB>(provider, range.clone())?;

// Default name doesn't have any configuration
let default_name: PathBuf = SnapshotSegment::Receipts.filename(&range).into();
let new_name: PathBuf = SnapshotSegment::Receipts
.filename_with_configuration(filters, compression, &range)
.into();

std::fs::rename(default_name, new_name)?;

Ok(())
}
Expand Down Expand Up @@ -62,11 +74,10 @@ impl Command {

let mut row_indexes = tx_range.clone().collect::<Vec<_>>();

let path = SnapshotSegment::Receipts.filename_with_configuration(
filters,
compression,
&block_range,
);
let path = SnapshotSegment::Receipts
.filename_with_configuration(filters, compression, &block_range)
.into();

let provider = SnapshotProvider::default();
let jar_provider =
provider.get_segment_provider(SnapshotSegment::Receipts, self.from, Some(path))?;
Expand Down
40 changes: 25 additions & 15 deletions bin/reth/src/db/snapshots/transactions.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@ use reth_provider::{
TransactionsProvider, TransactionsProviderExt,
};
use reth_snapshot::{segments, segments::Segment};
use std::{path::Path, sync::Arc};
use std::{
path::{Path, PathBuf},
sync::Arc,
};

impl Command {
pub(crate) fn generate_transactions_snapshot<DB: Database>(
Expand All @@ -24,15 +27,24 @@ impl Command {
inclusion_filter: InclusionFilter,
phf: PerfectHashingFunction,
) -> eyre::Result<()> {
let segment = segments::Transactions::new(
compression,
if self.with_filters {
Filters::WithFilters(inclusion_filter, phf)
} else {
Filters::WithoutFilters
},
);
segment.snapshot::<DB>(provider, self.from..=(self.from + self.block_interval - 1))?;
let range = self.from..=(self.from + self.block_interval - 1);
let filters = if self.with_filters {
Filters::WithFilters(inclusion_filter, phf)
} else {
Filters::WithoutFilters
};

let segment = segments::Transactions::new(compression, filters);

segment.snapshot::<DB>(provider, range.clone())?;

// Default name doesn't have any configuration
let default_name: PathBuf = SnapshotSegment::Transactions.filename(&range).into();
let new_name: PathBuf = SnapshotSegment::Transactions
.filename_with_configuration(filters, compression, &range)
.into();

std::fs::rename(default_name, new_name)?;

Ok(())
}
Expand Down Expand Up @@ -62,11 +74,9 @@ impl Command {

let mut row_indexes = tx_range.clone().collect::<Vec<_>>();

let path = SnapshotSegment::Transactions.filename_with_configuration(
filters,
compression,
&block_range,
);
let path = SnapshotSegment::Transactions
.filename_with_configuration(filters, compression, &block_range)
.into();
let provider = SnapshotProvider::default();
let jar_provider =
provider.get_segment_provider(SnapshotSegment::Transactions, self.from, Some(path))?;
Expand Down
26 changes: 10 additions & 16 deletions crates/primitives/src/snapshot/segment.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use crate::{
BlockNumber, TxNumber,
};
use serde::{Deserialize, Serialize};
use std::{ops::RangeInclusive, path::PathBuf, str::FromStr};
use std::{ops::RangeInclusive, str::FromStr};
use strum::{AsRefStr, EnumString};

#[derive(
Expand Down Expand Up @@ -36,7 +36,7 @@ pub enum SnapshotSegment {

impl SnapshotSegment {
/// Returns the default configuration of the segment.
const fn config(&self) -> (Filters, Compression) {
pub const fn config(&self) -> (Filters, Compression) {
let default_config = (
Filters::WithFilters(InclusionFilter::Cuckoo, super::PerfectHashingFunction::Fmph),
Compression::Lz4,
Expand All @@ -50,19 +50,20 @@ impl SnapshotSegment {
}

/// Returns the default file name for the provided segment and range.
pub fn filename(&self, range: &RangeInclusive<BlockNumber>) -> PathBuf {
let (filters, compression) = self.config();
self.filename_with_configuration(filters, compression, range)
pub fn filename(&self, range: &RangeInclusive<BlockNumber>) -> String {
// ATTENTION: if changing the name format, be sure to reflect those changes in
// [`Self::parse_filename`.]
format!("snapshot_{}_{}_{}", self.as_ref(), range.start(), range.end(),)
}

/// Returns file name for the provided segment, filters, compression and range.
/// Returns file name for the provided segment and range, alongisde filters, compression.
pub fn filename_with_configuration(
&self,
filters: Filters,
compression: Compression,
range: &RangeInclusive<BlockNumber>,
) -> PathBuf {
let segment_name = self.as_ref();
) -> String {
let prefix = self.filename(range);

let filters_name = match filters {
Filters::WithFilters(inclusion_filter, phf) => {
Expand All @@ -73,14 +74,7 @@ impl SnapshotSegment {

// ATTENTION: if changing the name format, be sure to reflect those changes in
// [`Self::parse_filename`.]
format!(
"snapshot_{segment_name}_{}_{}_{}_{}",
range.start(),
range.end(),
filters_name,
compression.as_ref()
)
.into()
format!("{prefix}_{}_{}", filters_name, compression.as_ref())
}

/// Takes a filename and parses the [`SnapshotSegment`] and its inclusive range.
Expand Down
4 changes: 2 additions & 2 deletions crates/snapshot/src/segments/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use reth_primitives::{
BlockNumber, SnapshotSegment,
};
use reth_provider::{DatabaseProviderRO, TransactionsProviderExt};
use std::ops::RangeInclusive;
use std::{ops::RangeInclusive, path::Path};

pub(crate) type Rows<const COLUMNS: usize> = [Vec<Vec<u8>>; COLUMNS];

Expand Down Expand Up @@ -61,7 +61,7 @@ pub(crate) fn prepare_jar<DB: Database, const COLUMNS: usize>(
let tx_range = provider.transaction_range_by_block_range(block_range.clone())?;
let mut nippy_jar = NippyJar::new(
COLUMNS,
&segment.filename_with_configuration(filters, compression, &block_range),
Path::new(segment.filename(&block_range).as_str()),
SegmentHeader::new(block_range, tx_range, segment),
);

Expand Down

0 comments on commit e87c64d

Please sign in to comment.