From 179ec707d6d189cc5d759ed0e5c9ceaba486df8d Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 3 Oct 2024 00:03:18 +0200 Subject: [PATCH] replace compact with bincode on SealedHeader --- Cargo.lock | 1 + crates/optimism/storage/src/lib.rs | 6 +----- crates/primitives-traits/src/header/sealed.rs | 6 +++--- crates/stages/stages/Cargo.toml | 3 ++- crates/stages/stages/src/stages/headers.rs | 18 +++++++++++++----- crates/storage/db-api/src/models/mod.rs | 8 ++------ 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76297369f67d..e19f4a9d8e4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8819,6 +8819,7 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "assert_matches", + "bincode", "criterion", "futures-util", "itertools 0.13.0", diff --git a/crates/optimism/storage/src/lib.rs b/crates/optimism/storage/src/lib.rs index 3d728f18743e..d435ed1d884a 100644 --- a/crates/optimism/storage/src/lib.rs +++ b/crates/optimism/storage/src/lib.rs @@ -16,9 +16,7 @@ mod tests { CompactClientVersion, CompactU256, CompactU64, StoredBlockBodyIndices, StoredBlockOmmers, StoredBlockWithdrawals, }; - use reth_primitives::{ - Account, Receipt, ReceiptWithBloom, Requests, SealedHeader, Withdrawals, - }; + use reth_primitives::{Account, Receipt, ReceiptWithBloom, Requests, Withdrawals}; use reth_prune_types::{PruneCheckpoint, PruneMode, PruneSegment}; use reth_stages_types::{ AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint, ExecutionCheckpoint, @@ -43,7 +41,6 @@ mod tests { assert_eq!(PruneSegment::bitflag_encoded_bytes(), 1); assert_eq!(Receipt::bitflag_encoded_bytes(), 2); assert_eq!(ReceiptWithBloom::bitflag_encoded_bytes(), 0); - assert_eq!(SealedHeader::bitflag_encoded_bytes(), 0); assert_eq!(StageCheckpoint::bitflag_encoded_bytes(), 1); assert_eq!(StageUnitCheckpoint::bitflag_encoded_bytes(), 1); assert_eq!(StoredBlockBodyIndices::bitflag_encoded_bytes(), 1); @@ -70,7 +67,6 @@ mod tests { validate_bitflag_backwards_compat!(PruneSegment, UnusedBits::Zero); validate_bitflag_backwards_compat!(Receipt, UnusedBits::NotZero); validate_bitflag_backwards_compat!(ReceiptWithBloom, UnusedBits::Zero); - validate_bitflag_backwards_compat!(SealedHeader, UnusedBits::Zero); validate_bitflag_backwards_compat!(StageCheckpoint, UnusedBits::NotZero); validate_bitflag_backwards_compat!(StageUnitCheckpoint, UnusedBits::Zero); validate_bitflag_backwards_compat!(StoredBlockBodyIndices, UnusedBits::Zero); diff --git a/crates/primitives-traits/src/header/sealed.rs b/crates/primitives-traits/src/header/sealed.rs index 9b11f080a462..f2047e079c8d 100644 --- a/crates/primitives-traits/src/header/sealed.rs +++ b/crates/primitives-traits/src/header/sealed.rs @@ -7,13 +7,13 @@ use alloy_rlp::{Decodable, Encodable}; use bytes::BufMut; use core::mem; use derive_more::{AsRef, Deref}; -use reth_codecs::{add_arbitrary_tests, Compact}; +use reth_codecs::add_arbitrary_tests; use serde::{Deserialize, Serialize}; /// A [`Header`] that is sealed at a precalculated hash, use [`SealedHeader::unseal()`] if you want /// to modify header. -#[derive(Debug, Clone, PartialEq, Eq, Hash, AsRef, Deref, Serialize, Deserialize, Compact)] -#[add_arbitrary_tests(rlp, compact)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, AsRef, Deref, Serialize, Deserialize)] +#[add_arbitrary_tests(rlp)] pub struct SealedHeader { /// Locked Header hash. hash: BlockHash, diff --git a/crates/stages/stages/Cargo.toml b/crates/stages/stages/Cargo.toml index 9ed84160477f..3d4227d8a274 100644 --- a/crates/stages/stages/Cargo.toml +++ b/crates/stages/stages/Cargo.toml @@ -24,7 +24,7 @@ reth-evm.workspace = true reth-exex.workspace = true reth-network-p2p.workspace = true reth-primitives = { workspace = true, features = ["secp256k1"] } -reth-primitives-traits.workspace = true +reth-primitives-traits = { workspace = true, features = ["serde-bincode-compat"] } reth-provider.workspace = true reth-execution-types.workspace = true reth-prune.workspace = true @@ -52,6 +52,7 @@ itertools.workspace = true rayon.workspace = true num-traits = "0.2.15" tempfile = { workspace = true, optional = true } +bincode.workspace = true [dev-dependencies] # reth diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index 83771c4969d9..8497c84cbf7b 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -1,6 +1,5 @@ use alloy_primitives::{BlockHash, BlockNumber, B256}; use futures_util::StreamExt; -use reth_codecs::Compact; use reth_config::config::EtlConfig; use reth_consensus::Consensus; use reth_db::{tables, RawKey, RawTable, RawValue}; @@ -12,6 +11,7 @@ use reth_db_api::{ use reth_etl::Collector; use reth_network_p2p::headers::{downloader::HeaderDownloader, error::HeadersDownloaderError}; use reth_primitives::{SealedHeader, StaticFileSegment}; +use reth_primitives_traits::serde_bincode_compat; use reth_provider::{ providers::{StaticFileProvider, StaticFileWriter}, BlockHashReader, DBProvider, HeaderProvider, HeaderSyncGap, HeaderSyncGapProvider, @@ -54,8 +54,8 @@ pub struct HeaderStage { sync_gap: Option, /// ETL collector with `HeaderHash` -> `BlockNumber` hash_collector: Collector, - /// ETL collector with `BlockNumber` -> `SealedHeader` - header_collector: Collector, + /// ETL collector with `BlockNumber` -> `BincodeSealedHeader` + header_collector: Collector>, /// Returns true if the ETL collector has all necessary headers to fill the gap. is_etl_ready: bool, } @@ -121,7 +121,11 @@ where info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers"); } - let (sealed_header, _) = SealedHeader::from_compact(&header_buf, header_buf.len()); + let sealed_header: SealedHeader = + bincode::deserialize::>(&header_buf) + .map_err(|err| StageError::Fatal(Box::new(err)))? + .into(); + let (header, header_hash) = sealed_header.split(); if header.number == 0 { continue @@ -240,7 +244,11 @@ where let header_number = header.number; self.hash_collector.insert(header.hash(), header_number)?; - self.header_collector.insert(header_number, header)?; + self.header_collector.insert( + header_number, + bincode::serialize(&serde_bincode_compat::SealedHeader::from(&header)) + .map_err(|err| StageError::Fatal(Box::new(err)))?, + )?; // Headers are downloaded in reverse, so if we reach here, we know we have // filled the gap. diff --git a/crates/storage/db-api/src/models/mod.rs b/crates/storage/db-api/src/models/mod.rs index 2466a42c52cd..1acb9f8cf752 100644 --- a/crates/storage/db-api/src/models/mod.rs +++ b/crates/storage/db-api/src/models/mod.rs @@ -8,8 +8,7 @@ use alloy_genesis::GenesisAccount; use alloy_primitives::{Address, Log, B256, U256}; use reth_codecs::{add_arbitrary_tests, Compact}; use reth_primitives::{ - Account, Bytecode, Header, Receipt, Requests, SealedHeader, StorageEntry, - TransactionSignedNoHash, TxType, + Account, Bytecode, Header, Receipt, Requests, StorageEntry, TransactionSignedNoHash, TxType, }; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::StageCheckpoint; @@ -210,7 +209,6 @@ macro_rules! impl_compression_for_compact { } impl_compression_for_compact!( - SealedHeader, Header, Account, Log, @@ -315,7 +313,7 @@ mod tests { fn test_ensure_backwards_compatibility() { use super::*; use reth_codecs::{test_utils::UnusedBits, validate_bitflag_backwards_compat}; - use reth_primitives::{Account, Receipt, ReceiptWithBloom, SealedHeader, Withdrawals}; + use reth_primitives::{Account, Receipt, ReceiptWithBloom, Withdrawals}; use reth_prune_types::{PruneCheckpoint, PruneMode, PruneSegment}; use reth_stages_types::{ AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint, @@ -337,7 +335,6 @@ mod tests { assert_eq!(PruneSegment::bitflag_encoded_bytes(), 1); assert_eq!(Receipt::bitflag_encoded_bytes(), 1); assert_eq!(ReceiptWithBloom::bitflag_encoded_bytes(), 0); - assert_eq!(SealedHeader::bitflag_encoded_bytes(), 0); assert_eq!(StageCheckpoint::bitflag_encoded_bytes(), 1); assert_eq!(StageUnitCheckpoint::bitflag_encoded_bytes(), 1); assert_eq!(StoredBlockBodyIndices::bitflag_encoded_bytes(), 1); @@ -361,7 +358,6 @@ mod tests { validate_bitflag_backwards_compat!(PruneSegment, UnusedBits::Zero); validate_bitflag_backwards_compat!(Receipt, UnusedBits::Zero); validate_bitflag_backwards_compat!(ReceiptWithBloom, UnusedBits::Zero); - validate_bitflag_backwards_compat!(SealedHeader, UnusedBits::Zero); validate_bitflag_backwards_compat!(StageCheckpoint, UnusedBits::NotZero); validate_bitflag_backwards_compat!(StageUnitCheckpoint, UnusedBits::Zero); validate_bitflag_backwards_compat!(StoredBlockBodyIndices, UnusedBits::Zero);