From 3b7132bc0d4a11e347ae92db2fb2b98b31e02459 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Tue, 30 Apr 2024 19:49:08 +0300 Subject: [PATCH 001/227] Attestation superstruct changes for EIP 7549 (#5644) * update * experiment * superstruct changes * revert * superstruct changes * fix tests * indexed attestation * indexed attestation superstruct * updated TODOs --- .../beacon_chain/src/attestation_simulator.rs | 2 +- .../src/attestation_verification.rs | 85 +++--- .../src/attestation_verification/batch.rs | 8 +- .../beacon_chain/src/beacon_block_reward.rs | 2 +- beacon_node/beacon_chain/src/beacon_chain.rs | 30 +- beacon_node/beacon_chain/src/block_reward.rs | 2 +- .../beacon_chain/src/early_attester_cache.rs | 6 +- .../src/naive_aggregation_pool.rs | 44 +-- .../beacon_chain/src/observed_aggregates.rs | 20 +- .../beacon_chain/src/observed_operations.rs | 6 +- beacon_node/beacon_chain/src/test_utils.rs | 62 ++-- .../beacon_chain/src/validator_monitor.rs | 26 +- .../http_api/src/block_packing_efficiency.rs | 42 +-- beacon_node/http_api/src/lib.rs | 12 +- .../http_api/src/publish_attestations.rs | 2 +- beacon_node/http_api/tests/tests.rs | 21 +- .../lighthouse_network/src/types/pubsub.rs | 8 +- .../gossip_methods.rs | 18 +- .../src/network_beacon_processor/mod.rs | 2 +- .../src/subnet_service/attestation_subnets.rs | 2 +- beacon_node/operation_pool/src/attestation.rs | 10 +- .../operation_pool/src/attestation_storage.rs | 84 +++++- beacon_node/operation_pool/src/lib.rs | 10 +- beacon_node/operation_pool/src/persistence.rs | 2 +- beacon_node/store/src/consensus_context.rs | 9 +- consensus/fork_choice/src/fork_choice.rs | 39 ++- consensus/fork_choice/tests/tests.rs | 24 +- .../src/common/get_attesting_indices.rs | 10 +- .../src/common/get_indexed_attestation.rs | 16 +- .../state_processing/src/consensus_context.rs | 39 ++- .../block_signature_verifier.rs | 2 +- .../is_valid_indexed_attestation.rs | 6 +- .../process_operations.rs | 53 ++-- .../per_block_processing/signature_sets.rs | 24 +- .../src/per_block_processing/tests.rs | 34 ++- .../verify_attestation.rs | 6 +- .../verify_attester_slashing.rs | 7 +- .../state_processing/src/verify_operation.rs | 4 +- consensus/types/src/aggregate_and_proof.rs | 6 +- consensus/types/src/attestation.rs | 274 +++++++++++++++++- consensus/types/src/beacon_block.rs | 26 +- consensus/types/src/eth_spec.rs | 8 + consensus/types/src/indexed_attestation.rs | 131 ++++++++- .../types/src/signed_aggregate_and_proof.rs | 2 +- lcli/src/indexed_attestations.rs | 2 +- slasher/src/array.rs | 20 +- slasher/src/attestation_queue.rs | 7 +- slasher/src/attester_record.rs | 10 +- slasher/src/config.rs | 3 +- slasher/src/database.rs | 6 +- slasher/src/slasher.rs | 6 +- slasher/src/test_utils.rs | 22 +- testing/web3signer_tests/src/lib.rs | 20 +- validator_client/src/attestation_service.rs | 20 +- .../src/http_api/tests/keystores.rs | 10 +- validator_client/src/validator_store.rs | 14 +- 56 files changed, 940 insertions(+), 426 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_simulator.rs b/beacon_node/beacon_chain/src/attestation_simulator.rs index 6453158458a..c97c4490af0 100644 --- a/beacon_node/beacon_chain/src/attestation_simulator.rs +++ b/beacon_node/beacon_chain/src/attestation_simulator.rs @@ -82,7 +82,7 @@ pub fn produce_unaggregated_attestation( // Store the unaggregated attestation in the validator monitor for later processing match chain.produce_unaggregated_attestation(current_slot, beacon_committee_index) { Ok(unaggregated_attestation) => { - let data = &unaggregated_attestation.data; + let data = unaggregated_attestation.data(); debug!( chain.log, diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index f3bde8678e1..131932febba 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -329,7 +329,7 @@ pub trait VerifiedAttestation: Sized { // Inefficient default implementation. This is overridden for gossip verified attestations. fn into_attestation_and_indices(self) -> (Attestation, Vec) { let attestation = self.attestation().clone(); - let attesting_indices = self.indexed_attestation().attesting_indices.clone().into(); + let attesting_indices = self.indexed_attestation().attesting_indices_to_vec(); (attestation, attesting_indices) } } @@ -455,17 +455,17 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { verify_propagation_slot_range(&chain.slot_clock, attestation, &chain.spec)?; // Check the attestation's epoch matches its target. - if attestation.data.slot.epoch(T::EthSpec::slots_per_epoch()) - != attestation.data.target.epoch + if attestation.data().slot.epoch(T::EthSpec::slots_per_epoch()) + != attestation.data().target.epoch { return Err(Error::InvalidTargetEpoch { - slot: attestation.data.slot, - epoch: attestation.data.target.epoch, + slot: attestation.data().slot, + epoch: attestation.data().target.epoch, }); } // Ensure the valid aggregated attestation has not already been seen locally. - let attestation_data = &attestation.data; + let attestation_data = attestation.data(); let attestation_data_root = attestation_data.tree_hash_root(); if chain @@ -486,7 +486,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { match chain .observed_aggregators .read() - .validator_has_been_observed(attestation.data.target.epoch, aggregator_index as usize) + .validator_has_been_observed(attestation.data().target.epoch, aggregator_index as usize) { Ok(true) => Err(Error::AggregatorAlreadyKnown(aggregator_index)), Ok(false) => Ok(()), @@ -518,7 +518,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { verify_attestation_target_root::(&head_block, attestation)?; // Ensure that the attestation has participants. - if attestation.aggregation_bits.is_zero() { + if attestation.is_aggregation_bits_zero() { Err(Error::EmptyAggregationBitfield) } else { Ok(attestation_data_root) @@ -611,12 +611,12 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { if chain .observed_aggregators .write() - .observe_validator(attestation.data.target.epoch, aggregator_index as usize) + .observe_validator(attestation.data().target.epoch, aggregator_index as usize) .map_err(BeaconChainError::from)? { return Err(Error::PriorAttestationKnown { validator_index: aggregator_index, - epoch: attestation.data.target.epoch, + epoch: attestation.data().target.epoch, }); } @@ -712,13 +712,13 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { attestation: &Attestation, chain: &BeaconChain, ) -> Result<(), Error> { - let attestation_epoch = attestation.data.slot.epoch(T::EthSpec::slots_per_epoch()); + let attestation_epoch = attestation.data().slot.epoch(T::EthSpec::slots_per_epoch()); // Check the attestation's epoch matches its target. - if attestation_epoch != attestation.data.target.epoch { + if attestation_epoch != attestation.data().target.epoch { return Err(Error::InvalidTargetEpoch { - slot: attestation.data.slot, - epoch: attestation.data.target.epoch, + slot: attestation.data().slot, + epoch: attestation.data().target.epoch, }); } @@ -730,7 +730,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { // Check to ensure that the attestation is "unaggregated". I.e., it has exactly one // aggregation bit set. - let num_aggregation_bits = attestation.aggregation_bits.num_set_bits(); + let num_aggregation_bits = attestation.num_set_aggregation_bits(); if num_aggregation_bits != 1 { return Err(Error::NotExactlyOneAggregationBitSet(num_aggregation_bits)); } @@ -757,7 +757,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { chain: &BeaconChain, ) -> Result<(u64, SubnetId), Error> { let expected_subnet_id = SubnetId::compute_subnet_for_attestation_data::( - &indexed_attestation.data, + indexed_attestation.data(), committees_per_slot, &chain.spec, ) @@ -774,8 +774,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { }; let validator_index = *indexed_attestation - .attesting_indices - .first() + .attesting_indices_first() .ok_or(Error::NotExactlyOneAggregationBitSet(0))?; /* @@ -785,12 +784,12 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { if chain .observed_gossip_attesters .read() - .validator_has_been_observed(attestation.data.target.epoch, validator_index as usize) + .validator_has_been_observed(attestation.data().target.epoch, validator_index as usize) .map_err(BeaconChainError::from)? { return Err(Error::PriorAttestationKnown { validator_index, - epoch: attestation.data.target.epoch, + epoch: attestation.data().target.epoch, }); } @@ -881,12 +880,12 @@ impl<'a, T: BeaconChainTypes> VerifiedUnaggregatedAttestation<'a, T> { if chain .observed_gossip_attesters .write() - .observe_validator(attestation.data.target.epoch, validator_index as usize) + .observe_validator(attestation.data().target.epoch, validator_index as usize) .map_err(BeaconChainError::from)? { return Err(Error::PriorAttestationKnown { validator_index, - epoch: attestation.data.target.epoch, + epoch: attestation.data().target.epoch, }); } Ok(()) @@ -998,28 +997,28 @@ fn verify_head_block_is_known( let block_opt = chain .canonical_head .fork_choice_read_lock() - .get_block(&attestation.data.beacon_block_root) + .get_block(&attestation.data().beacon_block_root) .or_else(|| { chain .early_attester_cache - .get_proto_block(attestation.data.beacon_block_root) + .get_proto_block(attestation.data().beacon_block_root) }); if let Some(block) = block_opt { // Reject any block that exceeds our limit on skipped slots. if let Some(max_skip_slots) = max_skip_slots { - if attestation.data.slot > block.slot + max_skip_slots { + if attestation.data().slot > block.slot + max_skip_slots { return Err(Error::TooManySkippedSlots { head_block_slot: block.slot, - attestation_slot: attestation.data.slot, + attestation_slot: attestation.data().slot, }); } } Ok(block) - } else if chain.is_pre_finalization_block(attestation.data.beacon_block_root)? { + } else if chain.is_pre_finalization_block(attestation.data().beacon_block_root)? { Err(Error::HeadBlockFinalized { - beacon_block_root: attestation.data.beacon_block_root, + beacon_block_root: attestation.data().beacon_block_root, }) } else { // The block is either: @@ -1029,7 +1028,7 @@ fn verify_head_block_is_known( // 2) A post-finalization block that we don't know about yet. We'll queue // the attestation until the block becomes available (or we time out). Err(Error::UnknownHeadBlock { - beacon_block_root: attestation.data.beacon_block_root, + beacon_block_root: attestation.data().beacon_block_root, }) } } @@ -1043,7 +1042,7 @@ pub fn verify_propagation_slot_range( attestation: &Attestation, spec: &ChainSpec, ) -> Result<(), Error> { - let attestation_slot = attestation.data.slot; + let attestation_slot = attestation.data().slot; let latest_permissible_slot = slot_clock .now_with_future_tolerance(spec.maximum_gossip_clock_disparity()) .ok_or(BeaconChainError::UnableToReadSlot)?; @@ -1095,11 +1094,11 @@ pub fn verify_attestation_signature( let fork = chain .spec - .fork_at_epoch(indexed_attestation.data.target.epoch); + .fork_at_epoch(indexed_attestation.data().target.epoch); let signature_set = indexed_attestation_signature_set_from_pubkeys( |validator_index| pubkey_cache.get(validator_index).map(Cow::Borrowed), - &indexed_attestation.signature, + indexed_attestation.signature(), indexed_attestation, &fork, chain.genesis_validators_root, @@ -1127,7 +1126,7 @@ pub fn verify_attestation_target_root( ) -> Result<(), Error> { // Check the attestation target root. let head_block_epoch = head_block.slot.epoch(E::slots_per_epoch()); - let attestation_epoch = attestation.data.slot.epoch(E::slots_per_epoch()); + let attestation_epoch = attestation.data().slot.epoch(E::slots_per_epoch()); if head_block_epoch > attestation_epoch { // The epoch references an invalid head block from a future epoch. // @@ -1140,7 +1139,7 @@ pub fn verify_attestation_target_root( // Reference: // https://github.com/ethereum/eth2.0-specs/pull/2001#issuecomment-699246659 return Err(Error::InvalidTargetRoot { - attestation: attestation.data.target.root, + attestation: attestation.data().target.root, // It is not clear what root we should expect in this case, since the attestation is // fundamentally invalid. expected: None, @@ -1159,9 +1158,9 @@ pub fn verify_attestation_target_root( }; // Reject any attestation with an invalid target root. - if target_root != attestation.data.target.root { + if target_root != attestation.data().target.root { return Err(Error::InvalidTargetRoot { - attestation: attestation.data.target.root, + attestation: attestation.data().target.root, expected: Some(target_root), }); } @@ -1199,7 +1198,7 @@ pub fn verify_signed_aggregate_signatures( let fork = chain .spec - .fork_at_epoch(indexed_attestation.data.target.epoch); + .fork_at_epoch(indexed_attestation.data().target.epoch); let signature_sets = vec![ signed_aggregate_selection_proof_signature_set( @@ -1220,7 +1219,7 @@ pub fn verify_signed_aggregate_signatures( .map_err(BeaconChainError::SignatureSetError)?, indexed_attestation_signature_set_from_pubkeys( |validator_index| pubkey_cache.get(validator_index).map(Cow::Borrowed), - &indexed_attestation.signature, + indexed_attestation.signature(), indexed_attestation, &fork, chain.genesis_validators_root, @@ -1266,8 +1265,8 @@ where T: BeaconChainTypes, F: Fn((BeaconCommittee, CommitteesPerSlot)) -> Result, { - let attestation_epoch = attestation.data.slot.epoch(T::EthSpec::slots_per_epoch()); - let target = &attestation.data.target; + let attestation_epoch = attestation.data().slot.epoch(T::EthSpec::slots_per_epoch()); + let target = &attestation.data().target; // Attestation target must be for a known block. // @@ -1290,12 +1289,12 @@ where let committees_per_slot = committee_cache.committees_per_slot(); Ok(committee_cache - .get_beacon_committee(attestation.data.slot, attestation.data.index) + .get_beacon_committee(attestation.data().slot, attestation.data().index) .map(|committee| map_fn((committee, committees_per_slot))) .unwrap_or_else(|| { Err(Error::NoCommitteeForSlotAndIndex { - slot: attestation.data.slot, - index: attestation.data.index, + slot: attestation.data().slot, + index: attestation.data().index, }) })) }) diff --git a/beacon_node/beacon_chain/src/attestation_verification/batch.rs b/beacon_node/beacon_chain/src/attestation_verification/batch.rs index 6aec2bef68a..1ec752ff5c5 100644 --- a/beacon_node/beacon_chain/src/attestation_verification/batch.rs +++ b/beacon_node/beacon_chain/src/attestation_verification/batch.rs @@ -73,7 +73,7 @@ where let indexed_attestation = &indexed.indexed_attestation; let fork = chain .spec - .fork_at_epoch(indexed_attestation.data.target.epoch); + .fork_at_epoch(indexed_attestation.data().target.epoch); signature_sets.push( signed_aggregate_selection_proof_signature_set( @@ -98,7 +98,7 @@ where signature_sets.push( indexed_attestation_signature_set_from_pubkeys( |validator_index| pubkey_cache.get(validator_index).map(Cow::Borrowed), - &indexed_attestation.signature, + indexed_attestation.signature(), indexed_attestation, &fork, chain.genesis_validators_root, @@ -182,11 +182,11 @@ where let indexed_attestation = &partially_verified.indexed_attestation; let fork = chain .spec - .fork_at_epoch(indexed_attestation.data.target.epoch); + .fork_at_epoch(indexed_attestation.data().target.epoch); let signature_set = indexed_attestation_signature_set_from_pubkeys( |validator_index| pubkey_cache.get(validator_index).map(Cow::Borrowed), - &indexed_attestation.signature, + indexed_attestation.signature(), indexed_attestation, &fork, chain.genesis_validators_root, diff --git a/beacon_node/beacon_chain/src/beacon_block_reward.rs b/beacon_node/beacon_chain/src/beacon_block_reward.rs index 5b70215d225..33567001e3c 100644 --- a/beacon_node/beacon_chain/src/beacon_block_reward.rs +++ b/beacon_node/beacon_chain/src/beacon_block_reward.rs @@ -202,7 +202,7 @@ impl BeaconChain { let mut previous_epoch_participation = state.previous_epoch_participation()?.clone(); for attestation in block.body().attestations() { - let data = &attestation.data; + let data = attestation.data(); let inclusion_delay = state.slot().safe_sub(data.slot)?.as_u64(); // [Modified in Deneb:EIP7045] let participation_flag_indices = get_attestation_participation_flag_indices( diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index dd1fe59b922..d8a546257be 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -120,6 +120,7 @@ use store::{ use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; use tree_hash::TreeHash; +use types::attestation::AttestationBase; use types::blob_sidecar::FixedBlobSidecarList; use types::payload::BlockProductionVersion; use types::*; @@ -1650,7 +1651,7 @@ impl BeaconChain { &self, attestation: Attestation, ) -> Result, Error> { - let beacon_block_root = attestation.data.beacon_block_root; + let beacon_block_root = attestation.data().beacon_block_root; match self .canonical_head .fork_choice_read_lock() @@ -1916,7 +1917,8 @@ impl BeaconChain { }; drop(cache_timer); - Ok(Attestation { + // TODO(electra) implement electra variant + Ok(Attestation::Base(AttestationBase { aggregation_bits: BitList::with_capacity(committee_len)?, data: AttestationData { slot: request_slot, @@ -1926,7 +1928,7 @@ impl BeaconChain { target, }, signature: AggregateSignature::empty(), - }) + })) } /// Performs the same validation as `Self::verify_unaggregated_attestation_for_gossip`, but for @@ -2138,8 +2140,8 @@ impl BeaconChain { self.log, "Stored unaggregated attestation"; "outcome" => ?outcome, - "index" => attestation.data.index, - "slot" => attestation.data.slot.as_u64(), + "index" => attestation.data().index, + "slot" => attestation.data().slot.as_u64(), ), Err(NaiveAggregationError::SlotTooLow { slot, @@ -2157,8 +2159,8 @@ impl BeaconChain { self.log, "Failed to store unaggregated attestation"; "error" => ?e, - "index" => attestation.data.index, - "slot" => attestation.data.slot.as_u64(), + "index" => attestation.data().index, + "slot" => attestation.data().slot.as_u64(), ); return Err(Error::from(e).into()); } @@ -3703,7 +3705,7 @@ impl BeaconChain { self.log, "Failed to get indexed attestation"; "purpose" => "validator monitor", - "attestation_slot" => attestation.data.slot, + "attestation_slot" => attestation.data().slot, "error" => ?e, ); continue; @@ -3759,7 +3761,7 @@ impl BeaconChain { self.log, "Failed to register observed attestation"; "error" => ?e, - "epoch" => a.data.target.epoch + "epoch" => a.data().target.epoch ); } } @@ -3771,7 +3773,7 @@ impl BeaconChain { self.log, "Failed to get indexed attestation"; "purpose" => "observation", - "attestation_slot" => a.data.slot, + "attestation_slot" => a.data().slot, "error" => ?e, ); continue; @@ -3780,15 +3782,15 @@ impl BeaconChain { let mut observed_block_attesters = self.observed_block_attesters.write(); - for &validator_index in &indexed_attestation.attesting_indices { + for &validator_index in indexed_attestation.attesting_indices_iter() { if let Err(e) = observed_block_attesters - .observe_validator(a.data.target.epoch, validator_index as usize) + .observe_validator(a.data().target.epoch, validator_index as usize) { debug!( self.log, "Failed to register observed block attester"; "error" => ?e, - "epoch" => a.data.target.epoch, + "epoch" => a.data().target.epoch, "validator_index" => validator_index, ) } @@ -3812,7 +3814,7 @@ impl BeaconChain { self.log, "Failed to get indexed attestation"; "purpose" => "slasher", - "attestation_slot" => attestation.data.slot, + "attestation_slot" => attestation.data().slot, "error" => ?e, ); continue; diff --git a/beacon_node/beacon_chain/src/block_reward.rs b/beacon_node/beacon_chain/src/block_reward.rs index fd0cfc7e9bd..44938668fe9 100644 --- a/beacon_node/beacon_chain/src/block_reward.rs +++ b/beacon_node/beacon_chain/src/block_reward.rs @@ -87,7 +87,7 @@ impl BeaconChain { .body() .attestations() .iter() - .map(|a| a.data.clone()) + .map(|a| a.data().clone()) .collect() } else { vec![] diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index 79d732f51b1..595f941235c 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -6,6 +6,7 @@ use crate::{ use parking_lot::RwLock; use proto_array::Block as ProtoBlock; use std::sync::Arc; +use types::attestation::AttestationBase; use types::*; pub struct CacheItem { @@ -122,7 +123,8 @@ impl EarlyAttesterCache { item.committee_lengths .get_committee_length::(request_slot, request_index, spec)?; - let attestation = Attestation { + // TODO(electra) make fork-agnostic + let attestation = Attestation::Base(AttestationBase { aggregation_bits: BitList::with_capacity(committee_len) .map_err(BeaconStateError::from)?, data: AttestationData { @@ -133,7 +135,7 @@ impl EarlyAttesterCache { target: item.target, }, signature: AggregateSignature::empty(), - }; + }); metrics::inc_counter(&metrics::BEACON_EARLY_ATTESTER_CACHE_HITS); diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index 7eeb9bb56f6..0eab13a4c1d 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -124,13 +124,22 @@ impl AggregateMap for AggregatedAttestationMap { fn insert(&mut self, a: &Self::Value) -> Result { let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_CORE_INSERT); - let set_bits = a - .aggregation_bits - .iter() - .enumerate() - .filter(|(_i, bit)| *bit) - .map(|(i, _bit)| i) - .collect::>(); + let set_bits = match a { + Attestation::Base(att) => att + .aggregation_bits + .iter() + .enumerate() + .filter(|(_i, bit)| *bit) + .map(|(i, _bit)| i) + .collect::>(), + Attestation::Electra(att) => att + .aggregation_bits + .iter() + .enumerate() + .filter(|(_i, bit)| *bit) + .map(|(i, _bit)| i) + .collect::>(), + }; let committee_index = set_bits .first() @@ -141,12 +150,11 @@ impl AggregateMap for AggregatedAttestationMap { return Err(Error::MoreThanOneAggregationBitSet(set_bits.len())); } - let attestation_data_root = a.data.tree_hash_root(); + let attestation_data_root = a.data().tree_hash_root(); if let Some(existing_attestation) = self.map.get_mut(&attestation_data_root) { if existing_attestation - .aggregation_bits - .get(committee_index) + .get_aggregation_bit(committee_index) .map_err(|_| Error::InconsistentBitfieldLengths)? { Ok(InsertOutcome::SignatureAlreadyKnown { committee_index }) @@ -476,8 +484,9 @@ mod tests { fn get_attestation(slot: Slot) -> Attestation { let mut a: Attestation = test_random_instance(); - a.data.slot = slot; - a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); + a.data_mut().slot = slot; + *a.aggregation_bits_base_mut().unwrap() = + BitList::with_capacity(4).expect("should create bitlist"); a } @@ -521,7 +530,8 @@ mod tests { } fn unset_attestation_bit(a: &mut Attestation, i: usize) { - a.aggregation_bits + a.aggregation_bits_base_mut() + .unwrap() .set(i, false) .expect("should unset aggregation bit") } @@ -533,19 +543,19 @@ mod tests { } fn mutate_attestation_block_root(a: &mut Attestation, block_root: Hash256) { - a.data.beacon_block_root = block_root + a.data_mut().beacon_block_root = block_root } fn mutate_attestation_slot(a: &mut Attestation, slot: Slot) { - a.data.slot = slot + a.data_mut().slot = slot } fn attestation_block_root_comparator(a: &Attestation, block_root: Hash256) -> bool { - a.data.beacon_block_root == block_root + a.data().beacon_block_root == block_root } fn key_from_attestation(a: &Attestation) -> AttestationData { - a.data.clone() + a.data().clone() } fn mutate_sync_contribution_block_root( diff --git a/beacon_node/beacon_chain/src/observed_aggregates.rs b/beacon_node/beacon_chain/src/observed_aggregates.rs index ab00aefcd3e..a83eb620788 100644 --- a/beacon_node/beacon_chain/src/observed_aggregates.rs +++ b/beacon_node/beacon_chain/src/observed_aggregates.rs @@ -105,21 +105,33 @@ pub trait SubsetItem { impl SubsetItem for Attestation { type Item = BitList; fn is_subset(&self, other: &Self::Item) -> bool { - self.aggregation_bits.is_subset(other) + match self { + Attestation::Base(att) => att.aggregation_bits.is_subset(other), + // TODO(electra) implement electra variant + Attestation::Electra(_) => todo!(), + } } fn is_superset(&self, other: &Self::Item) -> bool { - other.is_subset(&self.aggregation_bits) + match self { + Attestation::Base(att) => other.is_subset(&att.aggregation_bits), + // TODO(electra) implement electra variant + Attestation::Electra(_) => todo!(), + } } /// Returns the sync contribution aggregation bits. fn get_item(&self) -> Self::Item { - self.aggregation_bits.clone() + match self { + Attestation::Base(att) => att.aggregation_bits.clone(), + // TODO(electra) implement electra variant + Attestation::Electra(_) => todo!(), + } } /// Returns the hash tree root of the attestation data. fn root(&self) -> Hash256 { - self.data.tree_hash_root() + self.data().tree_hash_root() } } diff --git a/beacon_node/beacon_chain/src/observed_operations.rs b/beacon_node/beacon_chain/src/observed_operations.rs index 04861fbe318..a61cc4f74cc 100644 --- a/beacon_node/beacon_chain/src/observed_operations.rs +++ b/beacon_node/beacon_chain/src/observed_operations.rs @@ -63,14 +63,12 @@ impl ObservableOperation for AttesterSlashing { fn observed_validators(&self) -> SmallVec<[u64; SMALL_VEC_SIZE]> { let attestation_1_indices = self .attestation_1 - .attesting_indices - .iter() + .attesting_indices_iter() .copied() .collect::>(); let attestation_2_indices = self .attestation_2 - .attesting_indices - .iter() + .attesting_indices_iter() .copied() .collect::>(); attestation_1_indices diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index d3eeff8a7d9..4a53f64fcac 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -58,6 +58,8 @@ use store::{config::StoreConfig, HotColdDB, ItemStore, LevelDB, MemoryStore}; use task_executor::TaskExecutor; use task_executor::{test_utils::TestRuntime, ShutdownReason}; use tree_hash::TreeHash; +use types::attestation::AttestationBase; +use types::indexed_attestation::IndexedAttestationBase; use types::payload::BlockProductionVersion; pub use types::test_utils::generate_deterministic_keypairs; use types::test_utils::TestRandom; @@ -1030,7 +1032,8 @@ where *state.get_block_root(target_slot)? }; - Ok(Attestation { + // TODO(electra) make test fork-agnostic + Ok(Attestation::Base(AttestationBase { aggregation_bits: BitList::with_capacity(committee_len)?, data: AttestationData { slot, @@ -1043,7 +1046,7 @@ where }, }, signature: AggregateSignature::empty(), - }) + })) } /// A list of attestations for each committee for the given slot. @@ -1118,17 +1121,21 @@ where ) .unwrap(); - attestation.aggregation_bits.set(i, true).unwrap(); + attestation + .aggregation_bits_base_mut() + .unwrap() + .set(i, true) + .unwrap(); - attestation.signature = { + *attestation.signature_mut() = { let domain = self.spec.get_domain( - attestation.data.target.epoch, + attestation.data().target.epoch, Domain::BeaconAttester, &fork, state.genesis_validators_root(), ); - let message = attestation.data.signing_root(domain); + let message = attestation.data().signing_root(domain); let mut agg_sig = AggregateSignature::infinity(); @@ -1140,7 +1147,7 @@ where }; let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - &attestation.data, + attestation.data(), committee_count, &self.chain.spec, ) @@ -1314,7 +1321,7 @@ where // If there are any attestations in this committee, create an aggregate. if let Some((attestation, _)) = committee_attestations.first() { let bc = state - .get_beacon_committee(attestation.data.slot, attestation.data.index) + .get_beacon_committee(attestation.data().slot, attestation.data().index) .unwrap(); // Find an aggregator if one exists. Return `None` if there are no @@ -1345,7 +1352,7 @@ where // aggregate locally. let aggregate = self .chain - .get_aggregated_attestation(&attestation.data) + .get_aggregated_attestation(attestation.data()) .unwrap() .unwrap_or_else(|| { committee_attestations.iter().skip(1).fold( @@ -1485,7 +1492,7 @@ where ) -> AttesterSlashing { let fork = self.chain.canonical_head.cached_head().head_fork(); - let mut attestation_1 = IndexedAttestation { + let mut attestation_1 = IndexedAttestation::Base(IndexedAttestationBase { attesting_indices: VariableList::new(validator_indices).unwrap(), data: AttestationData { slot: Slot::new(0), @@ -1501,28 +1508,29 @@ where }, }, signature: AggregateSignature::infinity(), - }; + }); let mut attestation_2 = attestation_1.clone(); - attestation_2.data.index += 1; - attestation_2.data.source.epoch = source2.unwrap_or(Epoch::new(0)); - attestation_2.data.target.epoch = target2.unwrap_or(fork.epoch); + attestation_2.data_mut().index += 1; + attestation_2.data_mut().source.epoch = source2.unwrap_or(Epoch::new(0)); + attestation_2.data_mut().target.epoch = target2.unwrap_or(fork.epoch); for attestation in &mut [&mut attestation_1, &mut attestation_2] { - for &i in &attestation.attesting_indices { + // TODO(electra) we could explore iter mut here + for i in attestation.attesting_indices_to_vec() { let sk = &self.validator_keypairs[i as usize].sk; let genesis_validators_root = self.chain.genesis_validators_root; let domain = self.chain.spec.get_domain( - attestation.data.target.epoch, + attestation.data().target.epoch, Domain::BeaconAttester, &fork, genesis_validators_root, ); - let message = attestation.data.signing_root(domain); + let message = attestation.data().signing_root(domain); - attestation.signature.add_assign(&sk.sign(message)); + attestation.signature_mut().add_assign(&sk.sign(message)); } } @@ -1551,36 +1559,36 @@ where }, }; - let mut attestation_1 = IndexedAttestation { + let mut attestation_1 = IndexedAttestation::Base(IndexedAttestationBase { attesting_indices: VariableList::new(validator_indices_1).unwrap(), data: data.clone(), signature: AggregateSignature::infinity(), - }; + }); - let mut attestation_2 = IndexedAttestation { + let mut attestation_2 = IndexedAttestation::Base(IndexedAttestationBase { attesting_indices: VariableList::new(validator_indices_2).unwrap(), data, signature: AggregateSignature::infinity(), - }; + }); - attestation_2.data.index += 1; + attestation_2.data_mut().index += 1; let fork = self.chain.canonical_head.cached_head().head_fork(); for attestation in &mut [&mut attestation_1, &mut attestation_2] { - for &i in &attestation.attesting_indices { + for i in attestation.attesting_indices_to_vec() { let sk = &self.validator_keypairs[i as usize].sk; let genesis_validators_root = self.chain.genesis_validators_root; let domain = self.chain.spec.get_domain( - attestation.data.target.epoch, + attestation.data().target.epoch, Domain::BeaconAttester, &fork, genesis_validators_root, ); - let message = attestation.data.signing_root(domain); + let message = attestation.data().signing_root(domain); - attestation.signature.add_assign(&sk.sign(message)); + attestation.signature_mut().add_assign(&sk.sign(message)); } } diff --git a/beacon_node/beacon_chain/src/validator_monitor.rs b/beacon_node/beacon_chain/src/validator_monitor.rs index a63940074b4..70f86ac138e 100644 --- a/beacon_node/beacon_chain/src/validator_monitor.rs +++ b/beacon_node/beacon_chain/src/validator_monitor.rs @@ -469,7 +469,7 @@ impl ValidatorMonitor { unaggregated_attestations.remove(&oldest_slot); } } - let slot = attestation.data.slot; + let slot = attestation.data().slot; self.unaggregated_attestations.insert(slot, attestation); } @@ -730,12 +730,12 @@ impl ValidatorMonitor { // that qualifies the committee index for reward is included let inclusion_delay = spec.min_attestation_inclusion_delay; - let data = &unaggregated_attestation.data; + let data = unaggregated_attestation.data(); // Get the reward indices for the unaggregated attestation or log an error match get_attestation_participation_flag_indices( state, - &unaggregated_attestation.data, + unaggregated_attestation.data(), inclusion_delay, spec, ) { @@ -1233,7 +1233,7 @@ impl ValidatorMonitor { indexed_attestation: &IndexedAttestation, slot_clock: &S, ) { - let data = &indexed_attestation.data; + let data = indexed_attestation.data(); let epoch = data.slot.epoch(E::slots_per_epoch()); let delay = get_message_delay_ms( seen_timestamp, @@ -1242,7 +1242,7 @@ impl ValidatorMonitor { slot_clock, ); - indexed_attestation.attesting_indices.iter().for_each(|i| { + indexed_attestation.attesting_indices_iter().for_each(|i| { if let Some(validator) = self.get_validator(*i) { let id = &validator.id; @@ -1321,7 +1321,7 @@ impl ValidatorMonitor { indexed_attestation: &IndexedAttestation, slot_clock: &S, ) { - let data = &indexed_attestation.data; + let data = indexed_attestation.data(); let epoch = data.slot.epoch(E::slots_per_epoch()); let delay = get_message_delay_ms( seen_timestamp, @@ -1365,7 +1365,7 @@ impl ValidatorMonitor { }); } - indexed_attestation.attesting_indices.iter().for_each(|i| { + indexed_attestation.attesting_indices_iter().for_each(|i| { if let Some(validator) = self.get_validator(*i) { let id = &validator.id; @@ -1414,7 +1414,7 @@ impl ValidatorMonitor { parent_slot: Slot, spec: &ChainSpec, ) { - let data = &indexed_attestation.data; + let data = indexed_attestation.data(); // Best effort inclusion distance which ignores skip slots between the parent // and the current block. Skipped slots between the attestation slot and the parent // slot are still counted for simplicity's sake. @@ -1423,7 +1423,7 @@ impl ValidatorMonitor { let delay = inclusion_distance - spec.min_attestation_inclusion_delay; let epoch = data.slot.epoch(E::slots_per_epoch()); - indexed_attestation.attesting_indices.iter().for_each(|i| { + indexed_attestation.attesting_indices_iter().for_each(|i| { if let Some(validator) = self.get_validator(*i) { let id = &validator.id; @@ -1798,18 +1798,16 @@ impl ValidatorMonitor { } fn register_attester_slashing(&self, src: &str, slashing: &AttesterSlashing) { - let data = &slashing.attestation_1.data; + let data = slashing.attestation_1.data(); let attestation_1_indices: HashSet = slashing .attestation_1 - .attesting_indices - .iter() + .attesting_indices_iter() .copied() .collect(); slashing .attestation_2 - .attesting_indices - .iter() + .attesting_indices_iter() .filter(|index| attestation_1_indices.contains(index)) .filter_map(|index| self.get_validator(*index)) .for_each(|validator| { diff --git a/beacon_node/http_api/src/block_packing_efficiency.rs b/beacon_node/http_api/src/block_packing_efficiency.rs index f105fdf0a7d..3e511c25dff 100644 --- a/beacon_node/http_api/src/block_packing_efficiency.rs +++ b/beacon_node/http_api/src/block_packing_efficiency.rs @@ -10,8 +10,8 @@ use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::sync::Arc; use types::{ - BeaconCommittee, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, Epoch, EthSpec, - Hash256, OwnedBeaconCommittee, RelativeEpoch, SignedBeaconBlock, Slot, + Attestation, BeaconCommittee, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, Epoch, + EthSpec, Hash256, OwnedBeaconCommittee, RelativeEpoch, SignedBeaconBlock, Slot, }; use warp_utils::reject::{beacon_chain_error, custom_bad_request, custom_server_error}; @@ -112,21 +112,29 @@ impl PackingEfficiencyHandler { let mut attestations_in_block = HashMap::new(); for attestation in attestations.iter() { - for (position, voted) in attestation.aggregation_bits.iter().enumerate() { - if voted { - let unique_attestation = UniqueAttestation { - slot: attestation.data.slot, - committee_index: attestation.data.index, - committee_position: position, - }; - let inclusion_distance: u64 = block - .slot() - .as_u64() - .checked_sub(attestation.data.slot.as_u64()) - .ok_or(PackingEfficiencyError::InvalidAttestationError)?; - - self.available_attestations.remove(&unique_attestation); - attestations_in_block.insert(unique_attestation, inclusion_distance); + match attestation { + Attestation::Base(attn) => { + for (position, voted) in attn.aggregation_bits.iter().enumerate() { + if voted { + let unique_attestation = UniqueAttestation { + slot: attn.data.slot, + committee_index: attn.data.index, + committee_position: position, + }; + let inclusion_distance: u64 = block + .slot() + .as_u64() + .checked_sub(attn.data.slot.as_u64()) + .ok_or(PackingEfficiencyError::InvalidAttestationError)?; + + self.available_attestations.remove(&unique_attestation); + attestations_in_block.insert(unique_attestation, inclusion_distance); + } + } + } + // TODO(electra) implement electra variant + Attestation::Electra(_) => { + todo!() } } } diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 46d3ad7569b..87f7df07310 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1799,7 +1799,7 @@ pub fn serve( .naive_aggregation_pool .read() .iter() - .filter(|&att| query_filter(&att.data)) + .filter(|&att| query_filter(att.data())) .cloned(), ); Ok(api_types::GenericResponse::from(attestations)) @@ -3162,7 +3162,7 @@ pub fn serve( chain .produce_unaggregated_attestation(query.slot, query.committee_index) - .map(|attestation| attestation.data) + .map(|attestation| attestation.data().clone()) .map(api_types::GenericResponse::from) .map_err(warp_utils::reject::beacon_chain_error) }) @@ -3364,8 +3364,8 @@ pub fn serve( "error" => format!("{:?}", e), "request_index" => index, "aggregator_index" => aggregate.message.aggregator_index, - "attestation_index" => aggregate.message.aggregate.data.index, - "attestation_slot" => aggregate.message.aggregate.data.slot, + "attestation_index" => aggregate.message.aggregate.data().index, + "attestation_slot" => aggregate.message.aggregate.data().slot, ); failures.push(api_types::Failure::new(index, format!("Verification: {:?}", e))); } @@ -3385,8 +3385,8 @@ pub fn serve( "error" => format!("{:?}", e), "request_index" => index, "aggregator_index" => verified_aggregate.aggregate().message.aggregator_index, - "attestation_index" => verified_aggregate.attestation().data.index, - "attestation_slot" => verified_aggregate.attestation().data.slot, + "attestation_index" => verified_aggregate.attestation().data().index, + "attestation_slot" => verified_aggregate.attestation().data().slot, ); failures.push(api_types::Failure::new(index, format!("Fork choice: {:?}", e))); } diff --git a/beacon_node/http_api/src/publish_attestations.rs b/beacon_node/http_api/src/publish_attestations.rs index ed7f1ed17c9..8eaee093c1a 100644 --- a/beacon_node/http_api/src/publish_attestations.rs +++ b/beacon_node/http_api/src/publish_attestations.rs @@ -141,7 +141,7 @@ pub async fn publish_attestations( // move the `attestations` vec into the blocking task, so this small overhead is unavoidable. let attestation_metadata = attestations .iter() - .map(|att| (att.data.slot, att.data.index)) + .map(|att| (att.data().slot, att.data().index)) .collect::>(); // Gossip validate and publish attestations that can be immediately processed. diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index d44b9a688ce..21d93f22f33 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -35,8 +35,8 @@ use tokio::time::Duration; use tree_hash::TreeHash; use types::application_domain::ApplicationDomain; use types::{ - AggregateSignature, BitList, Domain, EthSpec, ExecutionBlockHash, Hash256, Keypair, - MainnetEthSpec, RelativeEpoch, SelectionProof, SignedRoot, Slot, + attestation::AttestationBase, AggregateSignature, BitList, Domain, EthSpec, ExecutionBlockHash, + Hash256, Keypair, MainnetEthSpec, RelativeEpoch, SelectionProof, SignedRoot, Slot, }; type E = MainnetEthSpec; @@ -1669,7 +1669,7 @@ impl ApiTester { let mut attestations = Vec::new(); for attestation in &self.attestations { let mut invalid_attestation = attestation.clone(); - invalid_attestation.data.slot += 1; + invalid_attestation.data_mut().slot += 1; // add both to ensure we only fail on invalid attestations attestations.push(attestation.clone()); @@ -1793,7 +1793,7 @@ impl ApiTester { pub async fn test_post_beacon_pool_attester_slashings_invalid(mut self) -> Self { let mut slashing = self.attester_slashing.clone(); - slashing.attestation_1.data.slot += 1; + slashing.attestation_1.data_mut().slot += 1; self.client .post_beacon_pool_attester_slashings(&slashing) @@ -3168,7 +3168,8 @@ impl ApiTester { .chain .produce_unaggregated_attestation(slot, index) .unwrap() - .data; + .data() + .clone(); assert_eq!(result, expected); } @@ -3188,8 +3189,8 @@ impl ApiTester { let result = self .client .get_validator_aggregate_attestation( - attestation.data.slot, - attestation.data.tree_hash_root(), + attestation.data().slot, + attestation.data().tree_hash_root(), ) .await .unwrap() @@ -3269,11 +3270,11 @@ impl ApiTester { .unwrap() .data; - let mut attestation = Attestation { + let mut attestation = Attestation::Base(AttestationBase { aggregation_bits: BitList::with_capacity(duty.committee_length as usize).unwrap(), data: attestation_data, signature: AggregateSignature::infinity(), - }; + }); attestation .sign( @@ -3312,7 +3313,7 @@ impl ApiTester { pub async fn test_get_validator_aggregate_and_proofs_invalid(mut self) -> Self { let mut aggregate = self.get_aggregate().await; - aggregate.message.aggregate.data.slot += 1; + aggregate.message.aggregate.data_mut().slot += 1; self.client .post_validator_aggregate_and_proof::(&[aggregate]) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index a5cf03412d5..653e264349e 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -343,14 +343,16 @@ impl std::fmt::Display for PubsubMessage { PubsubMessage::AggregateAndProofAttestation(att) => write!( f, "Aggregate and Proof: slot: {}, index: {}, aggregator_index: {}", - att.message.aggregate.data.slot, - att.message.aggregate.data.index, + att.message.aggregate.data().slot, + att.message.aggregate.data().index, att.message.aggregator_index, ), PubsubMessage::Attestation(data) => write!( f, "Attestation: subnet_id: {}, attestation_slot: {}, attestation_index: {}", - *data.0, data.1.data.slot, data.1.data.index, + *data.0, + data.1.data().slot, + data.1.data().index, ), PubsubMessage::VoluntaryExit(_data) => write!(f, "Voluntary Exit"), PubsubMessage::ProposerSlashing(_data) => write!(f, "Proposer Slashing"), diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 7b8826bd853..772c2efc944 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -72,7 +72,7 @@ impl VerifiedAttestation for VerifiedUnaggregate { fn into_attestation_and_indices(self) -> (Attestation, Vec) { let attestation = *self.attestation; - let attesting_indices = self.indexed_attestation.attesting_indices.into(); + let attesting_indices = self.indexed_attestation.attesting_indices_to_vec(); (attestation, attesting_indices) } } @@ -106,7 +106,7 @@ impl VerifiedAttestation for VerifiedAggregate { /// Efficient clone-free implementation that moves out of the `Box`. fn into_attestation_and_indices(self) -> (Attestation, Vec) { let attestation = self.signed_aggregate.message.aggregate; - let attesting_indices = self.indexed_attestation.attesting_indices.into(); + let attesting_indices = self.indexed_attestation.attesting_indices_to_vec(); (attestation, attesting_indices) } } @@ -133,7 +133,7 @@ enum FailedAtt { impl FailedAtt { pub fn beacon_block_root(&self) -> &Hash256 { - &self.attestation().data.beacon_block_root + &self.attestation().data().beacon_block_root } pub fn kind(&self) -> &'static str { @@ -309,7 +309,7 @@ impl NetworkBeaconProcessor { match result { Ok(verified_attestation) => { let indexed_attestation = &verified_attestation.indexed_attestation; - let beacon_block_root = indexed_attestation.data.beacon_block_root; + let beacon_block_root = indexed_attestation.data().beacon_block_root; // Register the attestation with any monitored validators. self.chain @@ -412,7 +412,7 @@ impl NetworkBeaconProcessor { reprocess_tx: Option>, seen_timestamp: Duration, ) { - let beacon_block_root = aggregate.message.aggregate.data.beacon_block_root; + let beacon_block_root = aggregate.message.aggregate.data().beacon_block_root; let result = match self .chain @@ -2282,7 +2282,7 @@ impl NetworkBeaconProcessor { self.log, "Ignored attestation to finalized block"; "block_root" => ?beacon_block_root, - "attestation_slot" => failed_att.attestation().data.slot, + "attestation_slot" => failed_att.attestation().data().slot, ); self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore); @@ -2305,9 +2305,9 @@ impl NetworkBeaconProcessor { debug!( self.log, "Dropping attestation"; - "target_root" => ?failed_att.attestation().data.target.root, + "target_root" => ?failed_att.attestation().data().target.root, "beacon_block_root" => ?beacon_block_root, - "slot" => ?failed_att.attestation().data.slot, + "slot" => ?failed_att.attestation().data().slot, "type" => ?attestation_type, "error" => ?e, "peer_id" => % peer_id @@ -2326,7 +2326,7 @@ impl NetworkBeaconProcessor { self.log, "Unable to validate attestation"; "beacon_block_root" => ?beacon_block_root, - "slot" => ?failed_att.attestation().data.slot, + "slot" => ?failed_att.attestation().data().slot, "type" => ?attestation_type, "peer_id" => %peer_id, "error" => ?e, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index f10646c7414..b85be1b0c15 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -144,7 +144,7 @@ impl NetworkBeaconProcessor { processor.process_gossip_aggregate_batch(aggregates, Some(reprocess_tx)) }; - let beacon_block_root = aggregate.message.aggregate.data.beacon_block_root; + let beacon_block_root = aggregate.message.aggregate.data().beacon_block_root; self.try_send(BeaconWorkEvent { drop_during_sync: true, work: Work::GossipAggregate { diff --git a/beacon_node/network/src/subnet_service/attestation_subnets.rs b/beacon_node/network/src/subnet_service/attestation_subnets.rs index ab9ffb95a6c..afe9815d674 100644 --- a/beacon_node/network/src/subnet_service/attestation_subnets.rs +++ b/beacon_node/network/src/subnet_service/attestation_subnets.rs @@ -389,7 +389,7 @@ impl AttestationService { .map(|tracked_vals| { tracked_vals.contains_key(&ExactSubnet { subnet_id: subnet, - slot: attestation.data.slot, + slot: attestation.data().slot, }) }) .unwrap_or(true) diff --git a/beacon_node/operation_pool/src/attestation.rs b/beacon_node/operation_pool/src/attestation.rs index 5c6f684e722..60074cde740 100644 --- a/beacon_node/operation_pool/src/attestation.rs +++ b/beacon_node/operation_pool/src/attestation.rs @@ -1,4 +1,4 @@ -use crate::attestation_storage::AttestationRef; +use crate::attestation_storage::{AttestationRef, CompactIndexedAttestation}; use crate::max_cover::MaxCover; use crate::reward_cache::RewardCache; use state_processing::common::{ @@ -83,7 +83,7 @@ impl<'a, E: EthSpec> AttMaxCover<'a, E> { let fresh_validators_rewards = att .indexed - .attesting_indices + .attesting_indices() .iter() .filter_map(|&index| { if reward_cache @@ -175,7 +175,11 @@ pub fn earliest_attestation_validators( base_state: &BeaconStateBase, ) -> BitList { // Bitfield of validators whose attestations are new/fresh. - let mut new_validators = attestation.indexed.aggregation_bits.clone(); + let mut new_validators = match attestation.indexed { + CompactIndexedAttestation::Base(indexed_att) => indexed_att.aggregation_bits.clone(), + // TODO(electra) per the comments above, this code path is obsolete post altair fork, so maybe we should just return an empty bitlist here? + CompactIndexedAttestation::Electra(_) => todo!(), + }; let state_attestations = if attestation.checkpoint.target_epoch == state.current_epoch() { &base_state.current_epoch_attestations diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index 43fdf3923bd..90fdf3cdb81 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -2,8 +2,8 @@ use crate::AttestationStats; use itertools::Itertools; use std::collections::HashMap; use types::{ - AggregateSignature, Attestation, AttestationData, BeaconState, BitList, Checkpoint, Epoch, - EthSpec, Hash256, Slot, + attestation::{AttestationBase, AttestationElectra}, superstruct, AggregateSignature, Attestation, AttestationData, + BeaconState, BitList, BitVector, Checkpoint, Epoch, EthSpec, Hash256, Slot, }; #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] @@ -20,11 +20,18 @@ pub struct CompactAttestationData { pub target_root: Hash256, } +#[superstruct(variants(Base, Electra), variant_attributes(derive(Debug, PartialEq,)))] #[derive(Debug, PartialEq)] pub struct CompactIndexedAttestation { pub attesting_indices: Vec, + #[superstruct(only(Base), partial_getter(rename = "aggregation_bits_base"))] pub aggregation_bits: BitList, + #[superstruct(only(Electra), partial_getter(rename = "aggregation_bits_electra"))] + pub aggregation_bits: BitList, pub signature: AggregateSignature, + pub index: u64, + #[superstruct(only(Electra))] + pub committee_bits: BitVector, } #[derive(Debug)] @@ -54,20 +61,29 @@ pub struct AttestationDataMap { impl SplitAttestation { pub fn new(attestation: Attestation, attesting_indices: Vec) -> Self { let checkpoint = CheckpointKey { - source: attestation.data.source, - target_epoch: attestation.data.target.epoch, + source: attestation.data().source, + target_epoch: attestation.data().target.epoch, }; let data = CompactAttestationData { - slot: attestation.data.slot, - index: attestation.data.index, - beacon_block_root: attestation.data.beacon_block_root, - target_root: attestation.data.target.root, + slot: attestation.data().slot, + index: attestation.data().index, + beacon_block_root: attestation.data().beacon_block_root, + target_root: attestation.data().target.root, }; - let indexed = CompactIndexedAttestation { - attesting_indices, - aggregation_bits: attestation.aggregation_bits, - signature: attestation.signature, + + let indexed = match attestation.clone() { + Attestation::Base(attn) => { + CompactIndexedAttestation::Base(CompactIndexedAttestationBase { + attesting_indices, + aggregation_bits: attn.aggregation_bits, + signature: attestation.signature().clone(), + index: data.index, + }) + } + // TODO(electra) implement electra variant + Attestation::Electra(_) => todo!(), }; + Self { checkpoint, data, @@ -99,10 +115,18 @@ impl<'a, E: EthSpec> AttestationRef<'a, E> { } pub fn clone_as_attestation(&self) -> Attestation { - Attestation { - aggregation_bits: self.indexed.aggregation_bits.clone(), - data: self.attestation_data(), - signature: self.indexed.signature.clone(), + match self.indexed { + CompactIndexedAttestation::Base(indexed_att) => Attestation::Base(AttestationBase { + aggregation_bits: indexed_att.aggregation_bits.clone(), + data: self.attestation_data(), + signature: indexed_att.signature.clone(), + }), + CompactIndexedAttestation::Electra(indexed_att) => Attestation::Electra(AttestationElectra { + aggregation_bits: indexed_att.aggregation_bits.clone(), + data: self.attestation_data(), + signature: indexed_att.signature.clone(), + committee_bits: indexed_att.committee_bits.clone(), + }), } } } @@ -125,6 +149,34 @@ impl CheckpointKey { } impl CompactIndexedAttestation { + pub fn signers_disjoint_from(&self, other: &Self) -> bool { + match (self, other) { + (CompactIndexedAttestation::Base(this), CompactIndexedAttestation::Base(other)) => { + this.signers_disjoint_from(other) + } + (CompactIndexedAttestation::Electra(_), CompactIndexedAttestation::Electra(_)) => { + todo!() + } + // TODO(electra) is a mix of electra and base compact indexed attestations an edge case we need to deal with? + _ => false, + } + } + + pub fn aggregate(&mut self, other: &Self) { + match (self, other) { + (CompactIndexedAttestation::Base(this), CompactIndexedAttestation::Base(other)) => { + this.aggregate(other) + } + (CompactIndexedAttestation::Electra(_), CompactIndexedAttestation::Electra(_)) => { + todo!() + } + // TODO(electra) is a mix of electra and base compact indexed attestations an edge case we need to deal with? + _ => (), + } + } +} + +impl CompactIndexedAttestationBase { pub fn signers_disjoint_from(&self, other: &Self) -> bool { self.aggregation_bits .intersection(&other.aggregation_bits) diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 6e744ccf62a..4b5fc259507 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -913,7 +913,7 @@ mod release_tests { let att2_split = SplitAttestation::new(att2.clone(), att2_indices); assert_eq!( - att1.aggregation_bits.num_set_bits(), + att1.num_set_aggregation_bits(), earliest_attestation_validators( &att1_split.as_ref(), &state, @@ -927,8 +927,8 @@ mod release_tests { .unwrap() .current_epoch_attestations .push(PendingAttestation { - aggregation_bits: att1.aggregation_bits.clone(), - data: att1.data.clone(), + aggregation_bits: att1.aggregation_bits_base().unwrap().clone(), + data: att1.data().clone(), inclusion_delay: 0, proposer_index: 0, }) @@ -1007,7 +1007,7 @@ mod release_tests { let agg_att = &block_attestations[0]; assert_eq!( - agg_att.aggregation_bits.num_set_bits(), + agg_att.num_set_aggregation_bits(), spec.target_committee_size as usize ); @@ -1243,7 +1243,7 @@ mod release_tests { // All the best attestations should be signed by at least `big_step_size` (4) validators. for att in &best_attestations { - assert!(att.aggregation_bits.num_set_bits() >= big_step_size); + assert!(att.num_set_aggregation_bits() >= big_step_size); } } diff --git a/beacon_node/operation_pool/src/persistence.rs b/beacon_node/operation_pool/src/persistence.rs index ef749a220db..8e4e5e2898d 100644 --- a/beacon_node/operation_pool/src/persistence.rs +++ b/beacon_node/operation_pool/src/persistence.rs @@ -76,7 +76,7 @@ impl PersistedOperationPool { .map(|att| { ( att.clone_as_attestation(), - att.indexed.attesting_indices.clone(), + att.indexed.attesting_indices().clone(), ) }) .collect(); diff --git a/beacon_node/store/src/consensus_context.rs b/beacon_node/store/src/consensus_context.rs index 08fad17b14b..727423d172f 100644 --- a/beacon_node/store/src/consensus_context.rs +++ b/beacon_node/store/src/consensus_context.rs @@ -21,8 +21,13 @@ pub struct OnDiskConsensusContext { /// /// They are not part of the on-disk format. #[ssz(skip_serializing, skip_deserializing)] - indexed_attestations: - HashMap<(AttestationData, BitList), IndexedAttestation>, + indexed_attestations: HashMap< + ( + AttestationData, + BitList, + ), + IndexedAttestation, + >, } impl OnDiskConsensusContext { diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 6e3f6717ede..f7836fb2fb4 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -241,10 +241,10 @@ pub struct QueuedAttestation { impl From<&IndexedAttestation> for QueuedAttestation { fn from(a: &IndexedAttestation) -> Self { Self { - slot: a.data.slot, - attesting_indices: a.attesting_indices[..].to_vec(), - block_root: a.data.beacon_block_root, - target_epoch: a.data.target.epoch, + slot: a.data().slot, + attesting_indices: a.attesting_indices_to_vec(), + block_root: a.data().beacon_block_root, + target_epoch: a.data().target.epoch, } } } @@ -948,20 +948,20 @@ where // // This is not in the specification, however it should be transparent to other nodes. We // return early here to avoid wasting precious resources verifying the rest of it. - if indexed_attestation.attesting_indices.is_empty() { + if indexed_attestation.attesting_indices_is_empty() { return Err(InvalidAttestation::EmptyAggregationBitfield); } - let target = indexed_attestation.data.target; + let target = indexed_attestation.data().target; if matches!(is_from_block, AttestationFromBlock::False) { self.validate_target_epoch_against_current_time(target.epoch)?; } - if target.epoch != indexed_attestation.data.slot.epoch(E::slots_per_epoch()) { + if target.epoch != indexed_attestation.data().slot.epoch(E::slots_per_epoch()) { return Err(InvalidAttestation::BadTargetEpoch { target: target.epoch, - slot: indexed_attestation.data.slot, + slot: indexed_attestation.data().slot, }); } @@ -983,9 +983,9 @@ where // attestation and do not delay consideration for later. let block = self .proto_array - .get_block(&indexed_attestation.data.beacon_block_root) + .get_block(&indexed_attestation.data().beacon_block_root) .ok_or(InvalidAttestation::UnknownHeadBlock { - beacon_block_root: indexed_attestation.data.beacon_block_root, + beacon_block_root: indexed_attestation.data().beacon_block_root, })?; // If an attestation points to a block that is from an earlier slot than the attestation, @@ -993,7 +993,7 @@ where // is from a prior epoch to the attestation, then the target root must be equal to the root // of the block that is being attested to. let expected_target = if target.epoch > block.slot.epoch(E::slots_per_epoch()) { - indexed_attestation.data.beacon_block_root + indexed_attestation.data().beacon_block_root } else { block.target_root }; @@ -1007,10 +1007,10 @@ where // Attestations must not be for blocks in the future. If this is the case, the attestation // should not be considered. - if block.slot > indexed_attestation.data.slot { + if block.slot > indexed_attestation.data().slot { return Err(InvalidAttestation::AttestsToFutureBlock { block: block.slot, - attestation: indexed_attestation.data.slot, + attestation: indexed_attestation.data().slot, }); } @@ -1055,18 +1055,18 @@ where // (1) becomes weird once we hit finality and fork choice drops the genesis block. (2) is // fine because votes to the genesis block are not useful; all validators implicitly attest // to genesis just by being present in the chain. - if attestation.data.beacon_block_root == Hash256::zero() { + if attestation.data().beacon_block_root == Hash256::zero() { return Ok(()); } self.validate_on_attestation(attestation, is_from_block)?; - if attestation.data.slot < self.fc_store.get_current_slot() { - for validator_index in attestation.attesting_indices.iter() { + if attestation.data().slot < self.fc_store.get_current_slot() { + for validator_index in attestation.attesting_indices_iter() { self.proto_array.process_attestation( *validator_index as usize, - attestation.data.beacon_block_root, - attestation.data.target.epoch, + attestation.data().beacon_block_root, + attestation.data().target.epoch, )?; } } else { @@ -1088,8 +1088,7 @@ where /// We assume that the attester slashing provided to this function has already been verified. pub fn on_attester_slashing(&mut self, slashing: &AttesterSlashing) { let attesting_indices_set = |att: &IndexedAttestation| { - att.attesting_indices - .iter() + att.attesting_indices_iter() .copied() .collect::>() }; diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index 3153275fb73..63bf3b51bd0 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -435,7 +435,7 @@ impl ForkChoiceTest { let validator_committee_index = 0; let validator_index = *head .beacon_state - .get_beacon_committee(current_slot, attestation.data.index) + .get_beacon_committee(current_slot, attestation.data().index) .expect("should get committees") .committee .get(validator_committee_index) @@ -831,7 +831,7 @@ async fn invalid_attestation_empty_bitfield() { .apply_attestation_to_chain( MutationDelay::NoDelay, |attestation, _| { - attestation.attesting_indices = vec![].into(); + *attestation.attesting_indices_base_mut().unwrap() = vec![].into(); }, |result| { assert_invalid_attestation!(result, InvalidAttestation::EmptyAggregationBitfield) @@ -853,7 +853,7 @@ async fn invalid_attestation_future_epoch() { .apply_attestation_to_chain( MutationDelay::NoDelay, |attestation, _| { - attestation.data.target.epoch = Epoch::new(2); + attestation.data_mut().target.epoch = Epoch::new(2); }, |result| { assert_invalid_attestation!( @@ -879,7 +879,7 @@ async fn invalid_attestation_past_epoch() { .apply_attestation_to_chain( MutationDelay::NoDelay, |attestation, _| { - attestation.data.target.epoch = Epoch::new(0); + attestation.data_mut().target.epoch = Epoch::new(0); }, |result| { assert_invalid_attestation!( @@ -903,7 +903,7 @@ async fn invalid_attestation_target_epoch() { .apply_attestation_to_chain( MutationDelay::NoDelay, |attestation, _| { - attestation.data.slot = Slot::new(1); + attestation.data_mut().slot = Slot::new(1); }, |result| { assert_invalid_attestation!( @@ -929,7 +929,7 @@ async fn invalid_attestation_unknown_target_root() { .apply_attestation_to_chain( MutationDelay::NoDelay, |attestation, _| { - attestation.data.target.root = junk; + attestation.data_mut().target.root = junk; }, |result| { assert_invalid_attestation!( @@ -955,7 +955,7 @@ async fn invalid_attestation_unknown_beacon_block_root() { .apply_attestation_to_chain( MutationDelay::NoDelay, |attestation, _| { - attestation.data.beacon_block_root = junk; + attestation.data_mut().beacon_block_root = junk; }, |result| { assert_invalid_attestation!( @@ -979,7 +979,7 @@ async fn invalid_attestation_future_block() { .apply_attestation_to_chain( MutationDelay::Blocks(1), |attestation, chain| { - attestation.data.beacon_block_root = chain + attestation.data_mut().beacon_block_root = chain .block_at_slot(chain.slot().unwrap(), WhenSlotSkipped::Prev) .unwrap() .unwrap() @@ -1010,13 +1010,13 @@ async fn invalid_attestation_inconsistent_ffg_vote() { .apply_attestation_to_chain( MutationDelay::NoDelay, |attestation, chain| { - attestation.data.target.root = chain + attestation.data_mut().target.root = chain .block_at_slot(Slot::new(1), WhenSlotSkipped::Prev) .unwrap() .unwrap() .canonical_root(); - *attestation_opt.lock().unwrap() = Some(attestation.data.target.root); + *attestation_opt.lock().unwrap() = Some(attestation.data().target.root); *local_opt.lock().unwrap() = Some( chain .block_at_slot(Slot::new(0), WhenSlotSkipped::Prev) @@ -1069,8 +1069,8 @@ async fn valid_attestation_skip_across_epoch() { MutationDelay::NoDelay, |attestation, _chain| { assert_eq!( - attestation.data.target.root, - attestation.data.beacon_block_root + attestation.data().target.root, + attestation.data().beacon_block_root ) }, |result| result.unwrap(), diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index a89b71ff2b5..6b8ce6f9372 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -27,6 +27,12 @@ pub fn get_attesting_indices_from_state( state: &BeaconState, att: &Attestation, ) -> Result, BeaconStateError> { - let committee = state.get_beacon_committee(att.data.slot, att.data.index)?; - get_attesting_indices::(committee.committee, &att.aggregation_bits) + let committee = state.get_beacon_committee(att.data().slot, att.data().index)?; + match att { + Attestation::Base(att) => { + get_attesting_indices::(committee.committee, &att.aggregation_bits) + } + // TODO(electra) implement get_attesting_indices for electra + Attestation::Electra(_) => todo!(), + } } diff --git a/consensus/state_processing/src/common/get_indexed_attestation.rs b/consensus/state_processing/src/common/get_indexed_attestation.rs index 9cf689df40f..d6c7512961c 100644 --- a/consensus/state_processing/src/common/get_indexed_attestation.rs +++ b/consensus/state_processing/src/common/get_indexed_attestation.rs @@ -1,6 +1,6 @@ use super::get_attesting_indices; use crate::per_block_processing::errors::{AttestationInvalid as Invalid, BlockOperationError}; -use types::*; +use types::{indexed_attestation::IndexedAttestationBase, *}; type Result = std::result::Result>; @@ -11,11 +11,15 @@ pub fn get_indexed_attestation( committee: &[usize], attestation: &Attestation, ) -> Result> { - let attesting_indices = get_attesting_indices::(committee, &attestation.aggregation_bits)?; + let attesting_indices = match attestation { + Attestation::Base(att) => get_attesting_indices::(committee, &att.aggregation_bits)?, + // TODO(electra) implement get_attesting_indices for electra + Attestation::Electra(_) => todo!(), + }; - Ok(IndexedAttestation { + Ok(IndexedAttestation::Base(IndexedAttestationBase { attesting_indices: VariableList::new(attesting_indices)?, - data: attestation.data.clone(), - signature: attestation.signature.clone(), - }) + data: attestation.data().clone(), + signature: attestation.signature().clone(), + })) } diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index 073d87be85b..b1196c942d3 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -21,8 +21,13 @@ pub struct ConsensusContext { /// Block root of the block at `slot`. pub current_block_root: Option, /// Cache of indexed attestations constructed during block processing. - pub indexed_attestations: - HashMap<(AttestationData, BitList), IndexedAttestation>, + pub indexed_attestations: HashMap< + ( + AttestationData, + BitList, + ), + IndexedAttestation, + >, } #[derive(Debug, PartialEq, Clone)] @@ -153,16 +158,29 @@ impl ConsensusContext { state: &BeaconState, attestation: &Attestation, ) -> Result<&IndexedAttestation, BlockOperationError> { - let key = ( - attestation.data.clone(), - attestation.aggregation_bits.clone(), - ); + let aggregation_bits = match attestation { + Attestation::Base(attn) => { + let mut extended_aggregation_bits: BitList = + BitList::with_capacity(attn.aggregation_bits.len()) + .map_err(BeaconStateError::from)?; + + for (i, bit) in attn.aggregation_bits.iter().enumerate() { + extended_aggregation_bits + .set(i, bit) + .map_err(BeaconStateError::from)?; + } + extended_aggregation_bits + } + Attestation::Electra(attn) => attn.aggregation_bits.clone(), + }; + + let key = (attestation.data().clone(), aggregation_bits); match self.indexed_attestations.entry(key) { Entry::Occupied(occupied) => Ok(occupied.into_mut()), Entry::Vacant(vacant) => { - let committee = - state.get_beacon_committee(attestation.data.slot, attestation.data.index)?; + let committee = state + .get_beacon_committee(attestation.data().slot, attestation.data().index)?; let indexed_attestation = get_indexed_attestation(committee.committee, attestation)?; Ok(vacant.insert(indexed_attestation)) @@ -178,7 +196,10 @@ impl ConsensusContext { pub fn set_indexed_attestations( mut self, attestations: HashMap< - (AttestationData, BitList), + ( + AttestationData, + BitList, + ), IndexedAttestation, >, ) -> Self { diff --git a/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs b/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs index 3b8a8ea52c9..884fe8305ba 100644 --- a/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs +++ b/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs @@ -290,7 +290,7 @@ where self.sets.push(indexed_attestation_signature_set( self.state, self.get_pubkey.clone(), - &attestation.signature, + attestation.signature(), indexed_attestation, self.spec, )?); diff --git a/consensus/state_processing/src/per_block_processing/is_valid_indexed_attestation.rs b/consensus/state_processing/src/per_block_processing/is_valid_indexed_attestation.rs index baccd1dbbd2..7c7c9e474ac 100644 --- a/consensus/state_processing/src/per_block_processing/is_valid_indexed_attestation.rs +++ b/consensus/state_processing/src/per_block_processing/is_valid_indexed_attestation.rs @@ -17,7 +17,7 @@ pub fn is_valid_indexed_attestation( verify_signatures: VerifySignatures, spec: &ChainSpec, ) -> Result<()> { - let indices = &indexed_attestation.attesting_indices; + let indices = indexed_attestation.attesting_indices_to_vec(); // Verify that indices aren't empty verify!(!indices.is_empty(), Invalid::IndicesEmpty); @@ -36,14 +36,14 @@ pub fn is_valid_indexed_attestation( })?; Ok(()) }; - check_sorted(indices)?; + check_sorted(&indices)?; if verify_signatures.is_true() { verify!( indexed_attestation_signature_set( state, |i| get_pubkey_from_state(state, i), - &indexed_attestation.signature, + indexed_attestation.signature(), indexed_attestation, spec )? diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index d812306a0f7..cb677f5514f 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -73,24 +73,33 @@ pub mod base { ) .map_err(|e| e.into_with_index(i))?; - let pending_attestation = PendingAttestation { - aggregation_bits: attestation.aggregation_bits.clone(), - data: attestation.data.clone(), - inclusion_delay: state.slot().safe_sub(attestation.data.slot)?.as_u64(), - proposer_index, + match attestation { + Attestation::Base(att) => { + let pending_attestation = PendingAttestation { + aggregation_bits: att.aggregation_bits.clone(), + data: att.data.clone(), + inclusion_delay: state.slot().safe_sub(att.data.slot)?.as_u64(), + proposer_index, + }; + + if attestation.data().target.epoch == state.current_epoch() { + state + .as_base_mut()? + .current_epoch_attestations + .push(pending_attestation)?; + } else { + state + .as_base_mut()? + .previous_epoch_attestations + .push(pending_attestation)?; + } + } + Attestation::Electra(_) => { + // TODO(electra) pending attestations are only phase 0 + // so we should just raise a relevant error here + todo!() + } }; - - if attestation.data.target.epoch == state.current_epoch() { - state - .as_base_mut()? - .current_epoch_attestations - .push(pending_attestation)?; - } else { - state - .as_base_mut()? - .previous_epoch_attestations - .push(pending_attestation)?; - } } Ok(()) @@ -128,26 +137,24 @@ pub mod altair_deneb { let previous_epoch = ctxt.previous_epoch; let current_epoch = ctxt.current_epoch; - let attesting_indices = verify_attestation_for_block_inclusion( + let indexed_att = verify_attestation_for_block_inclusion( state, attestation, ctxt, verify_signatures, spec, ) - .map_err(|e| e.into_with_index(att_index))? - .attesting_indices - .clone(); + .map_err(|e| e.into_with_index(att_index))?; // Matching roots, participation flag indices - let data = &attestation.data; + let data = attestation.data(); let inclusion_delay = state.slot().safe_sub(data.slot)?.as_u64(); let participation_flag_indices = get_attestation_participation_flag_indices(state, data, inclusion_delay, spec)?; // Update epoch participation flags. let mut proposer_reward_numerator = 0; - for index in &attesting_indices { + for index in indexed_att.attesting_indices_iter() { let index = *index as usize; let validator_effective_balance = state.epoch_cache().get_effective_balance(index)?; diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index 163b2cff7a9..19351a2c2f8 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -279,21 +279,21 @@ where E: EthSpec, F: Fn(usize) -> Option>, { - let mut pubkeys = Vec::with_capacity(indexed_attestation.attesting_indices.len()); - for &validator_idx in &indexed_attestation.attesting_indices { + let mut pubkeys = Vec::with_capacity(indexed_attestation.attesting_indices_len()); + for &validator_idx in indexed_attestation.attesting_indices_iter() { pubkeys.push( get_pubkey(validator_idx as usize).ok_or(Error::ValidatorUnknown(validator_idx))?, ); } let domain = spec.get_domain( - indexed_attestation.data.target.epoch, + indexed_attestation.data().target.epoch, Domain::BeaconAttester, &state.fork(), state.genesis_validators_root(), ); - let message = indexed_attestation.data.signing_root(domain); + let message = indexed_attestation.data().signing_root(domain); Ok(SignatureSet::multiple_pubkeys(signature, pubkeys, message)) } @@ -312,21 +312,21 @@ where E: EthSpec, F: Fn(usize) -> Option>, { - let mut pubkeys = Vec::with_capacity(indexed_attestation.attesting_indices.len()); - for &validator_idx in &indexed_attestation.attesting_indices { + let mut pubkeys = Vec::with_capacity(indexed_attestation.attesting_indices_len()); + for &validator_idx in indexed_attestation.attesting_indices_iter() { pubkeys.push( get_pubkey(validator_idx as usize).ok_or(Error::ValidatorUnknown(validator_idx))?, ); } let domain = spec.get_domain( - indexed_attestation.data.target.epoch, + indexed_attestation.data().target.epoch, Domain::BeaconAttester, fork, genesis_validators_root, ); - let message = indexed_attestation.data.signing_root(domain); + let message = indexed_attestation.data().signing_root(domain); Ok(SignatureSet::multiple_pubkeys(signature, pubkeys, message)) } @@ -346,14 +346,14 @@ where indexed_attestation_signature_set( state, get_pubkey.clone(), - &attester_slashing.attestation_1.signature, + attester_slashing.attestation_1.signature(), &attester_slashing.attestation_1, spec, )?, indexed_attestation_signature_set( state, get_pubkey, - &attester_slashing.attestation_2.signature, + attester_slashing.attestation_2.signature(), &attester_slashing.attestation_2, spec, )?, @@ -425,7 +425,7 @@ where E: EthSpec, F: Fn(usize) -> Option>, { - let slot = signed_aggregate_and_proof.message.aggregate.data.slot; + let slot = signed_aggregate_and_proof.message.aggregate.data().slot; let domain = spec.get_domain( slot.epoch(E::slots_per_epoch()), @@ -458,7 +458,7 @@ where let target_epoch = signed_aggregate_and_proof .message .aggregate - .data + .data() .target .epoch; diff --git a/consensus/state_processing/src/per_block_processing/tests.rs b/consensus/state_processing/src/per_block_processing/tests.rs index f0055fa80dd..85bb740a6cd 100644 --- a/consensus/state_processing/src/per_block_processing/tests.rs +++ b/consensus/state_processing/src/per_block_processing/tests.rs @@ -389,7 +389,7 @@ async fn invalid_attestation_no_committee_for_index() { .deconstruct() .0; head_block.to_mut().body_mut().attestations_mut()[0] - .data + .data_mut() .index += 1; let mut ctxt = ConsensusContext::new(state.slot()); let result = process_operations::process_attestations( @@ -423,11 +423,11 @@ async fn invalid_attestation_wrong_justified_checkpoint() { .clone() .deconstruct() .0; - let old_justified_checkpoint = head_block.body().attestations()[0].data.source; + let old_justified_checkpoint = head_block.body().attestations()[0].data().source; let mut new_justified_checkpoint = old_justified_checkpoint; new_justified_checkpoint.epoch += Epoch::new(1); head_block.to_mut().body_mut().attestations_mut()[0] - .data + .data_mut() .source = new_justified_checkpoint; let mut ctxt = ConsensusContext::new(state.slot()); @@ -467,8 +467,9 @@ async fn invalid_attestation_bad_aggregation_bitfield_len() { .clone() .deconstruct() .0; - head_block.to_mut().body_mut().attestations_mut()[0].aggregation_bits = - Bitfield::with_capacity(spec.target_committee_size).unwrap(); + *head_block.to_mut().body_mut().attestations_mut()[0] + .aggregation_bits_base_mut() + .unwrap() = Bitfield::with_capacity(spec.target_committee_size).unwrap(); let mut ctxt = ConsensusContext::new(state.slot()); let result = process_operations::process_attestations( @@ -501,7 +502,8 @@ async fn invalid_attestation_bad_signature() { .clone() .deconstruct() .0; - head_block.to_mut().body_mut().attestations_mut()[0].signature = AggregateSignature::empty(); + *head_block.to_mut().body_mut().attestations_mut()[0].signature_mut() = + AggregateSignature::empty(); let mut ctxt = ConsensusContext::new(state.slot()); let result = process_operations::process_attestations( @@ -536,10 +538,10 @@ async fn invalid_attestation_included_too_early() { .clone() .deconstruct() .0; - let new_attesation_slot = head_block.body().attestations()[0].data.slot + let new_attesation_slot = head_block.body().attestations()[0].data().slot + Slot::new(MainnetEthSpec::slots_per_epoch()); head_block.to_mut().body_mut().attestations_mut()[0] - .data + .data_mut() .slot = new_attesation_slot; let mut ctxt = ConsensusContext::new(state.slot()); @@ -579,10 +581,10 @@ async fn invalid_attestation_included_too_late() { .clone() .deconstruct() .0; - let new_attesation_slot = head_block.body().attestations()[0].data.slot + let new_attesation_slot = head_block.body().attestations()[0].data().slot - Slot::new(MainnetEthSpec::slots_per_epoch()); head_block.to_mut().body_mut().attestations_mut()[0] - .data + .data_mut() .slot = new_attesation_slot; let mut ctxt = ConsensusContext::new(state.slot()); @@ -620,7 +622,7 @@ async fn invalid_attestation_target_epoch_slot_mismatch() { .deconstruct() .0; head_block.to_mut().body_mut().attestations_mut()[0] - .data + .data_mut() .target .epoch += Epoch::new(1); @@ -699,7 +701,10 @@ async fn invalid_attester_slashing_1_invalid() { let harness = get_harness::(EPOCH_OFFSET, VALIDATOR_COUNT).await; let mut attester_slashing = harness.make_attester_slashing(vec![1, 2]); - attester_slashing.attestation_1.attesting_indices = VariableList::from(vec![2, 1]); + *attester_slashing + .attestation_1 + .attesting_indices_base_mut() + .unwrap() = VariableList::from(vec![2, 1]); let mut state = harness.get_current_state(); let mut ctxt = ConsensusContext::new(state.slot()); @@ -730,7 +735,10 @@ async fn invalid_attester_slashing_2_invalid() { let harness = get_harness::(EPOCH_OFFSET, VALIDATOR_COUNT).await; let mut attester_slashing = harness.make_attester_slashing(vec![1, 2]); - attester_slashing.attestation_2.attesting_indices = VariableList::from(vec![2, 1]); + *attester_slashing + .attestation_2 + .attesting_indices_base_mut() + .unwrap() = VariableList::from(vec![2, 1]); let mut state = harness.get_current_state(); let mut ctxt = ConsensusContext::new(state.slot()); diff --git a/consensus/state_processing/src/per_block_processing/verify_attestation.rs b/consensus/state_processing/src/per_block_processing/verify_attestation.rs index 73454559dfd..8369f988f73 100644 --- a/consensus/state_processing/src/per_block_processing/verify_attestation.rs +++ b/consensus/state_processing/src/per_block_processing/verify_attestation.rs @@ -22,7 +22,7 @@ pub fn verify_attestation_for_block_inclusion<'ctxt, E: EthSpec>( verify_signatures: VerifySignatures, spec: &ChainSpec, ) -> Result<&'ctxt IndexedAttestation> { - let data = &attestation.data; + let data = attestation.data(); verify!( data.slot.safe_add(spec.min_attestation_inclusion_delay)? <= state.slot(), @@ -66,7 +66,7 @@ pub fn verify_attestation_for_state<'ctxt, E: EthSpec>( verify_signatures: VerifySignatures, spec: &ChainSpec, ) -> Result<&'ctxt IndexedAttestation> { - let data = &attestation.data; + let data = attestation.data(); verify!( data.index < state.get_committee_count_at_slot(data.slot)?, @@ -90,7 +90,7 @@ fn verify_casper_ffg_vote( attestation: &Attestation, state: &BeaconState, ) -> Result<()> { - let data = &attestation.data; + let data = attestation.data(); verify!( data.target.epoch == data.slot.epoch(E::slots_per_epoch()), Invalid::TargetEpochSlotMismatch { diff --git a/consensus/state_processing/src/per_block_processing/verify_attester_slashing.rs b/consensus/state_processing/src/per_block_processing/verify_attester_slashing.rs index 0cb215fe93f..7fb784a968e 100644 --- a/consensus/state_processing/src/per_block_processing/verify_attester_slashing.rs +++ b/consensus/state_processing/src/per_block_processing/verify_attester_slashing.rs @@ -66,13 +66,12 @@ where let attestation_2 = &attester_slashing.attestation_2; let attesting_indices_1 = attestation_1 - .attesting_indices - .iter() + .attesting_indices_iter() .cloned() .collect::>(); + let attesting_indices_2 = attestation_2 - .attesting_indices - .iter() + .attesting_indices_iter() .cloned() .collect::>(); diff --git a/consensus/state_processing/src/verify_operation.rs b/consensus/state_processing/src/verify_operation.rs index b3924cd9732..4a2e5e22c56 100644 --- a/consensus/state_processing/src/verify_operation.rs +++ b/consensus/state_processing/src/verify_operation.rs @@ -159,8 +159,8 @@ impl VerifyOperation for AttesterSlashing { #[allow(clippy::arithmetic_side_effects)] fn verification_epochs(&self) -> SmallVec<[Epoch; MAX_FORKS_VERIFIED_AGAINST]> { smallvec![ - self.attestation_1.data.target.epoch, - self.attestation_2.data.target.epoch + self.attestation_1.data().target.epoch, + self.attestation_2.data().target.epoch ] } } diff --git a/consensus/types/src/aggregate_and_proof.rs b/consensus/types/src/aggregate_and_proof.rs index bfbf4d97afd..0297c6d6845 100644 --- a/consensus/types/src/aggregate_and_proof.rs +++ b/consensus/types/src/aggregate_and_proof.rs @@ -53,7 +53,7 @@ impl AggregateAndProof { let selection_proof = selection_proof .unwrap_or_else(|| { SelectionProof::new::( - aggregate.data.slot, + aggregate.data().slot, secret_key, fork, genesis_validators_root, @@ -77,14 +77,14 @@ impl AggregateAndProof { genesis_validators_root: Hash256, spec: &ChainSpec, ) -> bool { - let target_epoch = self.aggregate.data.slot.epoch(E::slots_per_epoch()); + let target_epoch = self.aggregate.data().slot.epoch(E::slots_per_epoch()); let domain = spec.get_domain( target_epoch, Domain::SelectionProof, fork, genesis_validators_root, ); - let message = self.aggregate.data.slot.signing_root(domain); + let message = self.aggregate.data().slot.signing_root(domain); self.selection_proof.verify(validator_pubkey, message) } } diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index e43077d0591..e036f958fff 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -1,13 +1,17 @@ +use crate::slot_data::SlotData; +use crate::{test_utils::TestRandom, Hash256, Slot}; use derivative::Derivative; +use rand::RngCore; use safe_arith::ArithError; use serde::{Deserialize, Serialize}; +use ssz::Decode; use ssz_derive::{Decode, Encode}; +use ssz_types::BitVector; +use std::hash::{Hash, Hasher}; +use superstruct::superstruct; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use crate::slot_data::SlotData; -use crate::{test_utils::TestRandom, Hash256, Slot}; - use super::{ AggregateSignature, AttestationData, BitList, ChainSpec, Domain, EthSpec, Fork, SecretKey, Signature, SignedRoot, @@ -20,31 +24,273 @@ pub enum Error { SubnetCountIsZero(ArithError), } -/// Details an attestation that can be slashable. -/// -/// Spec v0.12.1 +#[superstruct( + variants(Base, Electra), + variant_attributes( + derive( + Debug, + Clone, + Serialize, + Deserialize, + Decode, + Encode, + TestRandom, + Derivative, + arbitrary::Arbitrary, + TreeHash, + ), + derivative(PartialEq, Hash(bound = "E: EthSpec")), + serde(bound = "E: EthSpec", deny_unknown_fields), + arbitrary(bound = "E: EthSpec"), + ) +)] #[derive( - arbitrary::Arbitrary, Debug, Clone, Serialize, - Deserialize, - Encode, - Decode, TreeHash, - TestRandom, + Encode, Derivative, + Deserialize, + arbitrary::Arbitrary, + PartialEq, )] -#[derivative(PartialEq, Hash(bound = "E: EthSpec"))] -#[serde(bound = "E: EthSpec")] +#[serde(untagged)] +#[tree_hash(enum_behaviour = "transparent")] +#[ssz(enum_behaviour = "transparent")] +#[serde(bound = "E: EthSpec", deny_unknown_fields)] #[arbitrary(bound = "E: EthSpec")] pub struct Attestation { + #[superstruct(only(Base), partial_getter(rename = "aggregation_bits_base"))] pub aggregation_bits: BitList, + #[superstruct(only(Electra), partial_getter(rename = "aggregation_bits_electra"))] + pub aggregation_bits: BitList, pub data: AttestationData, pub signature: AggregateSignature, + #[superstruct(only(Electra))] + pub committee_bits: BitVector, +} + +impl Decode for Attestation { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + if let Ok(result) = AttestationBase::from_ssz_bytes(bytes) { + return Ok(Attestation::Base(result)); + } + + if let Ok(result) = AttestationElectra::from_ssz_bytes(bytes) { + return Ok(Attestation::Electra(result)); + } + + Err(ssz::DecodeError::BytesInvalid(String::from( + "bytes not valid for any fork variant", + ))) + } +} + +impl TestRandom for Attestation { + fn random_for_test(rng: &mut impl RngCore) -> Self { + let aggregation_bits: BitList = BitList::random_for_test(rng); + // let committee_bits: BitList = BitList::random_for_test(rng); + let data = AttestationData::random_for_test(rng); + let signature = AggregateSignature::random_for_test(rng); + + Self::Base(AttestationBase { + aggregation_bits, + // committee_bits, + data, + signature, + }) + } +} + +impl Hash for Attestation { + fn hash(&self, state: &mut H) + where + H: Hasher, + { + match self { + Attestation::Base(att) => att.hash(state), + Attestation::Electra(att) => att.hash(state), + } + } } impl Attestation { + /// Aggregate another Attestation into this one. + /// + /// The aggregation bitfields must be disjoint, and the data must be the same. + pub fn aggregate(&mut self, other: &Self) { + match self { + Attestation::Base(att) => { + debug_assert!(other.as_base().is_ok()); + + if let Ok(other) = other.as_base() { + att.aggregate(other) + } + } + Attestation::Electra(att) => { + debug_assert!(other.as_electra().is_ok()); + + if let Ok(other) = other.as_electra() { + att.aggregate(other) + } + } + } + } + + /// Signs `self`, setting the `committee_position`'th bit of `aggregation_bits` to `true`. + /// + /// Returns an `AlreadySigned` error if the `committee_position`'th bit is already `true`. + pub fn sign( + &mut self, + secret_key: &SecretKey, + committee_position: usize, + fork: &Fork, + genesis_validators_root: Hash256, + spec: &ChainSpec, + ) -> Result<(), Error> { + match self { + Attestation::Base(att) => att.sign( + secret_key, + committee_position, + fork, + genesis_validators_root, + spec, + ), + Attestation::Electra(att) => att.sign( + secret_key, + committee_position, + fork, + genesis_validators_root, + spec, + ), + } + } + + /// Returns an `AlreadySigned` error if the `committee_position`'th bit is already `true`. + pub fn add_signature( + &mut self, + signature: &Signature, + committee_position: usize, + ) -> Result<(), Error> { + match self { + Attestation::Base(att) => att.add_signature(signature, committee_position), + Attestation::Electra(att) => att.add_signature(signature, committee_position), + } + } + + pub fn committee_index(&self) -> u64 { + match self { + Attestation::Base(att) => att.data.index, + Attestation::Electra(att) => att.committee_index(), + } + } + + pub fn is_aggregation_bits_zero(&self) -> bool { + match self { + Attestation::Base(att) => att.aggregation_bits.is_zero(), + Attestation::Electra(att) => att.aggregation_bits.is_zero(), + } + } + + pub fn num_set_aggregation_bits(&self) -> usize { + match self { + Attestation::Base(att) => att.aggregation_bits.num_set_bits(), + Attestation::Electra(att) => att.aggregation_bits.num_set_bits(), + } + } + + pub fn get_aggregation_bit(&self, index: usize) -> Result { + match self { + Attestation::Base(att) => att.aggregation_bits.get(index), + Attestation::Electra(att) => att.aggregation_bits.get(index), + } + } +} + +impl AttestationElectra { + /// Are the aggregation bitfields of these attestations disjoint? + pub fn signers_disjoint_from(&self, other: &Self) -> bool { + self.aggregation_bits + .intersection(&other.aggregation_bits) + .is_zero() + } + + pub fn committee_index(&self) -> u64 { + *self.get_committee_indices().first().unwrap_or(&0u64) + } + + pub fn get_committee_indices(&self) -> Vec { + self.committee_bits + .iter() + .enumerate() + .filter_map(|(index, bit)| if bit { Some(index as u64) } else { None }) + .collect() + } + + /// Aggregate another Attestation into this one. + /// + /// The aggregation bitfields must be disjoint, and the data must be the same. + pub fn aggregate(&mut self, other: &Self) { + debug_assert_eq!(self.data, other.data); + debug_assert!(self.signers_disjoint_from(other)); + self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); + self.signature.add_assign_aggregate(&other.signature); + } + + /// Signs `self`, setting the `committee_position`'th bit of `aggregation_bits` to `true`. + /// + /// Returns an `AlreadySigned` error if the `committee_position`'th bit is already `true`. + pub fn sign( + &mut self, + secret_key: &SecretKey, + committee_position: usize, + fork: &Fork, + genesis_validators_root: Hash256, + spec: &ChainSpec, + ) -> Result<(), Error> { + let domain = spec.get_domain( + self.data.target.epoch, + Domain::BeaconAttester, + fork, + genesis_validators_root, + ); + let message = self.data.signing_root(domain); + + self.add_signature(&secret_key.sign(message), committee_position) + } + + /// Adds `signature` to `self` and sets the `committee_position`'th bit of `aggregation_bits` to `true`. + /// + /// Returns an `AlreadySigned` error if the `committee_position`'th bit is already `true`. + pub fn add_signature( + &mut self, + signature: &Signature, + committee_position: usize, + ) -> Result<(), Error> { + if self + .aggregation_bits + .get(committee_position) + .map_err(Error::SszTypesError)? + { + Err(Error::AlreadySigned(committee_position)) + } else { + self.aggregation_bits + .set(committee_position, true) + .map_err(Error::SszTypesError)?; + + self.signature.add_assign(signature); + + Ok(()) + } + } +} + +impl AttestationBase { /// Are the aggregation bitfields of these attestations disjoint? pub fn signers_disjoint_from(&self, other: &Self) -> bool { self.aggregation_bits @@ -113,7 +359,7 @@ impl Attestation { impl SlotData for Attestation { fn get_slot(&self) -> Slot { - self.data.slot + self.data().slot } } diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 14874f0204f..dd1a12abba1 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -1,3 +1,4 @@ +use crate::attestation::AttestationBase; use crate::test_utils::TestRandom; use crate::*; use derivative::Derivative; @@ -10,6 +11,8 @@ use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; +use self::indexed_attestation::IndexedAttestationBase; + /// A block of the `BeaconChain`. #[superstruct( variants(Base, Altair, Merge, Capella, Deneb, Electra), @@ -324,15 +327,16 @@ impl> BeaconBlockBase { message: header, signature: Signature::empty(), }; - let indexed_attestation: IndexedAttestation = IndexedAttestation { - attesting_indices: VariableList::new(vec![ - 0_u64; - E::MaxValidatorsPerCommittee::to_usize() - ]) - .unwrap(), - data: AttestationData::default(), - signature: AggregateSignature::empty(), - }; + let indexed_attestation: IndexedAttestation = + IndexedAttestation::Base(IndexedAttestationBase { + attesting_indices: VariableList::new(vec![ + 0_u64; + E::MaxValidatorsPerCommittee::to_usize() + ]) + .unwrap(), + data: AttestationData::default(), + signature: AggregateSignature::empty(), + }); let deposit_data = DepositData { pubkey: PublicKeyBytes::empty(), @@ -350,12 +354,12 @@ impl> BeaconBlockBase { attestation_2: indexed_attestation, }; - let attestation: Attestation = Attestation { + let attestation: Attestation = Attestation::Base(AttestationBase { aggregation_bits: BitList::with_capacity(E::MaxValidatorsPerCommittee::to_usize()) .unwrap(), data: AttestationData::default(), signature: AggregateSignature::empty(), - }; + }); let deposit = Deposit { proof: FixedVector::from_elem(Hash256::zero()), diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index a700c5e9abb..f9031f98908 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -63,6 +63,8 @@ pub trait EthSpec: * Misc */ type MaxValidatorsPerCommittee: Unsigned + Clone + Sync + Send + Debug + PartialEq + Eq; + type MaxValidatorsPerCommitteePerSlot: Unsigned + Clone + Sync + Send + Debug + PartialEq + Eq; + type MaxCommitteesPerSlot: Unsigned + Clone + Sync + Send + Debug + PartialEq + Eq; /* * Time parameters */ @@ -349,6 +351,8 @@ impl EthSpec for MainnetEthSpec { type JustificationBitsLength = U4; type SubnetBitfieldLength = U64; type MaxValidatorsPerCommittee = U2048; + type MaxCommitteesPerSlot = U64; + type MaxValidatorsPerCommitteePerSlot = U131072; type GenesisEpoch = U0; type SlotsPerEpoch = U32; type EpochsPerEth1VotingPeriod = U64; @@ -424,6 +428,8 @@ impl EthSpec for MinimalEthSpec { SubnetBitfieldLength, SyncCommitteeSubnetCount, MaxValidatorsPerCommittee, + MaxCommitteesPerSlot, + MaxValidatorsPerCommitteePerSlot, GenesisEpoch, HistoricalRootsLimit, ValidatorRegistryLimit, @@ -468,6 +474,8 @@ impl EthSpec for GnosisEthSpec { type JustificationBitsLength = U4; type SubnetBitfieldLength = U64; type MaxValidatorsPerCommittee = U2048; + type MaxCommitteesPerSlot = U64; + type MaxValidatorsPerCommitteePerSlot = U131072; type GenesisEpoch = U0; type SlotsPerEpoch = U16; type EpochsPerEth1VotingPeriod = U64; diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index d80b49d55a7..4477b236fad 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -1,9 +1,13 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, EthSpec, VariableList}; +use core::slice::Iter; use derivative::Derivative; +use rand::RngCore; use serde::{Deserialize, Serialize}; +use ssz::Decode; use ssz::Encode; use ssz_derive::{Decode, Encode}; use std::hash::{Hash, Hasher}; +use superstruct::superstruct; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; @@ -12,25 +16,50 @@ use tree_hash_derive::TreeHash; /// To be included in an `AttesterSlashing`. /// /// Spec v0.12.1 +#[superstruct( + variants(Base, Electra), + variant_attributes( + derive( + Debug, + Clone, + Serialize, + Deserialize, + Decode, + Encode, + TestRandom, + Derivative, + arbitrary::Arbitrary, + TreeHash, + ), + derivative(PartialEq, Hash(bound = "E: EthSpec")), + serde(bound = "E: EthSpec", deny_unknown_fields), + arbitrary(bound = "E: EthSpec"), + ) +)] #[derive( - Derivative, Debug, Clone, Serialize, - Deserialize, - Encode, - Decode, TreeHash, - TestRandom, + Encode, + Derivative, + Deserialize, arbitrary::Arbitrary, + PartialEq, )] -#[derivative(PartialEq, Eq)] // to satisfy Clippy's lint about `Hash` -#[serde(bound = "E: EthSpec")] +#[serde(untagged)] +#[tree_hash(enum_behaviour = "transparent")] +#[ssz(enum_behaviour = "transparent")] +#[serde(bound = "E: EthSpec", deny_unknown_fields)] #[arbitrary(bound = "E: EthSpec")] pub struct IndexedAttestation { /// Lists validator registry indices, not committee indices. + #[superstruct(only(Base), partial_getter(rename = "attesting_indices_base"))] #[serde(with = "quoted_variable_list_u64")] pub attesting_indices: VariableList, + #[superstruct(only(Electra), partial_getter(rename = "attesting_indices_electra"))] + #[serde(with = "quoted_variable_list_u64")] + pub attesting_indices: VariableList, pub data: AttestationData, pub signature: AggregateSignature, } @@ -40,15 +69,84 @@ impl IndexedAttestation { /// /// Spec v0.12.1 pub fn is_double_vote(&self, other: &Self) -> bool { - self.data.target.epoch == other.data.target.epoch && self.data != other.data + self.data().target.epoch == other.data().target.epoch && self.data() != other.data() } /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. /// /// Spec v0.12.1 pub fn is_surround_vote(&self, other: &Self) -> bool { - self.data.source.epoch < other.data.source.epoch - && other.data.target.epoch < self.data.target.epoch + self.data().source.epoch < other.data().source.epoch + && other.data().target.epoch < self.data().target.epoch + } + + pub fn attesting_indices_len(&self) -> usize { + match self { + IndexedAttestation::Base(att) => att.attesting_indices.len(), + IndexedAttestation::Electra(att) => att.attesting_indices.len(), + } + } + + pub fn attesting_indices_to_vec(&self) -> Vec { + match self { + IndexedAttestation::Base(att) => att.attesting_indices.to_vec(), + IndexedAttestation::Electra(att) => att.attesting_indices.to_vec(), + } + } + + pub fn attesting_indices_is_empty(&self) -> bool { + match self { + IndexedAttestation::Base(att) => att.attesting_indices.is_empty(), + IndexedAttestation::Electra(att) => att.attesting_indices.is_empty(), + } + } + + pub fn attesting_indices_iter(&self) -> Iter<'_, u64> { + match self { + IndexedAttestation::Base(att) => att.attesting_indices.iter(), + IndexedAttestation::Electra(att) => att.attesting_indices.iter(), + } + } + + pub fn attesting_indices_first(&self) -> Option<&u64> { + match self { + IndexedAttestation::Base(att) => att.attesting_indices.first(), + IndexedAttestation::Electra(att) => att.attesting_indices.first(), + } + } +} + +impl Decode for IndexedAttestation { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + if let Ok(result) = IndexedAttestationBase::from_ssz_bytes(bytes) { + return Ok(IndexedAttestation::Base(result)); + } + + if let Ok(result) = IndexedAttestationElectra::from_ssz_bytes(bytes) { + return Ok(IndexedAttestation::Electra(result)); + } + + Err(ssz::DecodeError::BytesInvalid(String::from( + "bytes not valid for any fork variant", + ))) + } +} + +impl TestRandom for IndexedAttestation { + fn random_for_test(rng: &mut impl RngCore) -> Self { + let attesting_indices = VariableList::random_for_test(rng); + let data = AttestationData::random_for_test(rng); + let signature = AggregateSignature::random_for_test(rng); + + Self::Base(IndexedAttestationBase { + attesting_indices, + data, + signature, + }) } } @@ -59,9 +157,12 @@ impl IndexedAttestation { /// Used in the operation pool. impl Hash for IndexedAttestation { fn hash(&self, state: &mut H) { - self.attesting_indices.hash(state); - self.data.hash(state); - self.signature.as_ssz_bytes().hash(state); + match self { + IndexedAttestation::Base(att) => att.attesting_indices.hash(state), + IndexedAttestation::Electra(att) => att.attesting_indices.hash(state), + }; + self.data().hash(state); + self.signature().as_ssz_bytes().hash(state); } } @@ -166,8 +267,8 @@ mod tests { let mut rng = XorShiftRng::from_seed([42; 16]); let mut indexed_vote = IndexedAttestation::random_for_test(&mut rng); - indexed_vote.data.source.epoch = Epoch::new(source_epoch); - indexed_vote.data.target.epoch = Epoch::new(target_epoch); + indexed_vote.data_mut().source.epoch = Epoch::new(source_epoch); + indexed_vote.data_mut().target.epoch = Epoch::new(target_epoch); indexed_vote } } diff --git a/consensus/types/src/signed_aggregate_and_proof.rs b/consensus/types/src/signed_aggregate_and_proof.rs index c31c50ea174..491edb3cb0d 100644 --- a/consensus/types/src/signed_aggregate_and_proof.rs +++ b/consensus/types/src/signed_aggregate_and_proof.rs @@ -57,7 +57,7 @@ impl SignedAggregateAndProof { spec, ); - let target_epoch = message.aggregate.data.slot.epoch(E::slots_per_epoch()); + let target_epoch = message.aggregate.data().slot.epoch(E::slots_per_epoch()); let domain = spec.get_domain( target_epoch, Domain::AggregateAndProof, diff --git a/lcli/src/indexed_attestations.rs b/lcli/src/indexed_attestations.rs index 63f8cd94637..4c8b06a5088 100644 --- a/lcli/src/indexed_attestations.rs +++ b/lcli/src/indexed_attestations.rs @@ -34,7 +34,7 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { let indexed_attestations = attestations .into_iter() .map(|att| { - let committee = state.get_beacon_committee(att.data.slot, att.data.index)?; + let committee = state.get_beacon_committee(att.data().slot, att.data().index)?; get_indexed_attestation(committee.committee, &att) }) .collect::, _>>() diff --git a/slasher/src/array.rs b/slasher/src/array.rs index b733b07c63f..77ddceb85fe 100644 --- a/slasher/src/array.rs +++ b/slasher/src/array.rs @@ -226,12 +226,12 @@ impl TargetArrayChunk for MinTargetChunk { ) -> Result, Error> { let min_target = self.chunk - .get_target(validator_index, attestation.data.source.epoch, config)?; - if attestation.data.target.epoch > min_target { + .get_target(validator_index, attestation.data().source.epoch, config)?; + if attestation.data().target.epoch > min_target { let existing_attestation = db.get_attestation_for_validator(txn, validator_index, min_target)?; - if attestation.data.source.epoch < existing_attestation.data.source.epoch { + if attestation.data().source.epoch < existing_attestation.data().source.epoch { Ok(AttesterSlashingStatus::SurroundsExisting(Box::new( existing_attestation, ))) @@ -329,12 +329,12 @@ impl TargetArrayChunk for MaxTargetChunk { ) -> Result, Error> { let max_target = self.chunk - .get_target(validator_index, attestation.data.source.epoch, config)?; - if attestation.data.target.epoch < max_target { + .get_target(validator_index, attestation.data().source.epoch, config)?; + if attestation.data().target.epoch < max_target { let existing_attestation = db.get_attestation_for_validator(txn, validator_index, max_target)?; - if existing_attestation.data.source.epoch < attestation.data.source.epoch { + if existing_attestation.data().source.epoch < attestation.data().source.epoch { Ok(AttesterSlashingStatus::SurroundedByExisting(Box::new( existing_attestation, ))) @@ -428,7 +428,7 @@ pub fn apply_attestation_for_validator( current_epoch: Epoch, config: &Config, ) -> Result, Error> { - let mut chunk_index = config.chunk_index(attestation.data.source.epoch); + let mut chunk_index = config.chunk_index(attestation.data().source.epoch); let mut current_chunk = get_chunk_for_update( db, txn, @@ -446,7 +446,7 @@ pub fn apply_attestation_for_validator( } let Some(mut start_epoch) = - T::first_start_epoch(attestation.data.source.epoch, current_epoch, config) + T::first_start_epoch(attestation.data().source.epoch, current_epoch, config) else { return Ok(slashing_status); }; @@ -465,7 +465,7 @@ pub fn apply_attestation_for_validator( chunk_index, validator_index, start_epoch, - attestation.data.target.epoch, + attestation.data().target.epoch, current_epoch, config, )?; @@ -492,7 +492,7 @@ pub fn update( let mut chunk_attestations = BTreeMap::new(); for attestation in batch { chunk_attestations - .entry(config.chunk_index(attestation.indexed.data.source.epoch)) + .entry(config.chunk_index(attestation.indexed.data().source.epoch)) .or_insert_with(Vec::new) .push(attestation); } diff --git a/slasher/src/attestation_queue.rs b/slasher/src/attestation_queue.rs index 3d23932df9f..62a1bb09455 100644 --- a/slasher/src/attestation_queue.rs +++ b/slasher/src/attestation_queue.rs @@ -47,7 +47,8 @@ impl AttestationBatch { self.attestations.push(Arc::downgrade(&indexed_record)); let attestation_data_hash = indexed_record.record.attestation_data_hash; - for &validator_index in &indexed_record.indexed.attesting_indices { + + for &validator_index in indexed_record.indexed.attesting_indices_iter() { self.attesters .entry((validator_index, attestation_data_hash)) .and_modify(|existing_entry| { @@ -56,8 +57,8 @@ impl AttestationBatch { // smaller indexed attestation. Single-bit attestations will usually be removed // completely by this process, and aggregates will only be retained if they // are not redundant with respect to a larger aggregate seen in the same batch. - if existing_entry.indexed.attesting_indices.len() - < indexed_record.indexed.attesting_indices.len() + if existing_entry.indexed.attesting_indices_len() + < indexed_record.indexed.attesting_indices_len() { *existing_entry = indexed_record.clone(); } diff --git a/slasher/src/attester_record.rs b/slasher/src/attester_record.rs index 56fdcb809ff..8de25160725 100644 --- a/slasher/src/attester_record.rs +++ b/slasher/src/attester_record.rs @@ -80,18 +80,20 @@ impl IndexedAttesterRecord { #[derive(Debug, Clone, Encode, Decode, TreeHash)] struct IndexedAttestationHeader { - pub attesting_indices: VariableList, + pub attesting_indices: VariableList, pub data_root: Hash256, pub signature: AggregateSignature, } impl From> for AttesterRecord { fn from(indexed_attestation: IndexedAttestation) -> AttesterRecord { - let attestation_data_hash = indexed_attestation.data.tree_hash_root(); + let attestation_data_hash = indexed_attestation.data().tree_hash_root(); + let attesting_indices = + VariableList::new(indexed_attestation.attesting_indices_to_vec()).unwrap_or_default(); let header = IndexedAttestationHeader:: { - attesting_indices: indexed_attestation.attesting_indices, + attesting_indices, data_root: attestation_data_hash, - signature: indexed_attestation.signature, + signature: indexed_attestation.signature().clone(), }; let indexed_attestation_hash = header.tree_hash_root(); AttesterRecord { diff --git a/slasher/src/config.rs b/slasher/src/config.rs index 4fd74343e76..26ac8841539 100644 --- a/slasher/src/config.rs +++ b/slasher/src/config.rs @@ -166,8 +166,7 @@ impl Config { validator_chunk_index: usize, ) -> impl Iterator + 'a { attestation - .attesting_indices - .iter() + .attesting_indices_iter() .filter(move |v| self.validator_chunk_index(**v) == validator_chunk_index) .copied() } diff --git a/slasher/src/database.rs b/slasher/src/database.rs index 49d2b00a4cd..d80112c553d 100644 --- a/slasher/src/database.rs +++ b/slasher/src/database.rs @@ -438,7 +438,7 @@ impl SlasherDB { ) -> Result { // Look-up ID by hash. let id_key = IndexedAttestationIdKey::new( - indexed_attestation.data.target.epoch, + indexed_attestation.data().target.epoch, indexed_attestation_hash, ); @@ -500,7 +500,7 @@ impl SlasherDB { // Otherwise, load the indexed attestation, compute the root and cache it. let indexed_attestation = self.get_indexed_attestation(txn, indexed_id)?; - let attestation_data_root = indexed_attestation.data.tree_hash_root(); + let attestation_data_root = indexed_attestation.data().tree_hash_root(); cache.put(indexed_id, attestation_data_root); @@ -536,7 +536,7 @@ impl SlasherDB { indexed_attestation_id: IndexedAttestationId, ) -> Result, Error> { // See if there's an existing attestation for this attester. - let target_epoch = attestation.data.target.epoch; + let target_epoch = attestation.data().target.epoch; let prev_max_target = self.get_attester_max_target(validator_index, txn)?; diff --git a/slasher/src/slasher.rs b/slasher/src/slasher.rs index 066c8d63d98..40b49fbbc66 100644 --- a/slasher/src/slasher.rs +++ b/slasher/src/slasher.rs @@ -299,7 +299,7 @@ impl Slasher { self.log, "Found double-vote slashing"; "validator_index" => validator_index, - "epoch" => slashing.attestation_1.data.target.epoch, + "epoch" => slashing.attestation_1.data().target.epoch, ); slashings.insert(slashing); } @@ -325,8 +325,8 @@ impl Slasher { for indexed_record in batch { let attestation = &indexed_record.indexed; - let target_epoch = attestation.data.target.epoch; - let source_epoch = attestation.data.source.epoch; + let target_epoch = attestation.data().target.epoch; + let source_epoch = attestation.data().source.epoch; if source_epoch > target_epoch || source_epoch + self.config.history_length as u64 <= current_epoch diff --git a/slasher/src/test_utils.rs b/slasher/src/test_utils.rs index 0011df1ffb0..c46b2f39492 100644 --- a/slasher/src/test_utils.rs +++ b/slasher/src/test_utils.rs @@ -1,7 +1,8 @@ use std::collections::HashSet; use types::{ - AggregateSignature, AttestationData, AttesterSlashing, BeaconBlockHeader, Checkpoint, Epoch, - Hash256, IndexedAttestation, MainnetEthSpec, Signature, SignedBeaconBlockHeader, Slot, + indexed_attestation::IndexedAttestationBase, AggregateSignature, AttestationData, + AttesterSlashing, BeaconBlockHeader, Checkpoint, Epoch, Hash256, IndexedAttestation, + MainnetEthSpec, Signature, SignedBeaconBlockHeader, Slot, }; pub type E = MainnetEthSpec; @@ -12,7 +13,8 @@ pub fn indexed_att( target_epoch: u64, target_root: u64, ) -> IndexedAttestation { - IndexedAttestation { + // TODO(electra) make fork-agnostic + IndexedAttestation::Base(IndexedAttestationBase { attesting_indices: attesting_indices.as_ref().to_vec().into(), data: AttestationData { slot: Slot::new(0), @@ -28,7 +30,7 @@ pub fn indexed_att( }, }, signature: AggregateSignature::empty(), - } + }) } pub fn att_slashing( @@ -66,7 +68,9 @@ pub fn slashed_validators_from_slashings(slashings: &HashSet "invalid slashing: {:#?}", slashing ); - hashset_intersection(&att1.attesting_indices, &att2.attesting_indices) + let attesting_indices_1 = att1.attesting_indices_to_vec(); + let attesting_indices_2 = att2.attesting_indices_to_vec(); + hashset_intersection(&attesting_indices_1, &attesting_indices_2) }) .collect() } @@ -82,10 +86,14 @@ pub fn slashed_validators_from_attestations( continue; } + // TODO(electra) in a nested loop, copying to vecs feels bad + let attesting_indices_1 = att1.attesting_indices_to_vec(); + let attesting_indices_2 = att2.attesting_indices_to_vec(); + if att1.is_double_vote(att2) || att1.is_surround_vote(att2) { slashed_validators.extend(hashset_intersection( - &att1.attesting_indices, - &att2.attesting_indices, + &attesting_indices_1, + &attesting_indices_2, )); } } diff --git a/testing/web3signer_tests/src/lib.rs b/testing/web3signer_tests/src/lib.rs index 3090b4da556..cf5d5f738d0 100644 --- a/testing/web3signer_tests/src/lib.rs +++ b/testing/web3signer_tests/src/lib.rs @@ -39,7 +39,7 @@ mod tests { use tempfile::{tempdir, TempDir}; use tokio::sync::OnceCell; use tokio::time::sleep; - use types::*; + use types::{attestation::AttestationBase, *}; use url::Url; use validator_client::{ initialized_validators::{ @@ -542,7 +542,7 @@ mod tests { /// Get a generic, arbitrary attestation for signing. fn get_attestation() -> Attestation { - Attestation { + Attestation::Base(AttestationBase { aggregation_bits: BitList::with_capacity(1).unwrap(), data: AttestationData { slot: <_>::default(), @@ -558,7 +558,7 @@ mod tests { }, }, signature: AggregateSignature::empty(), - } + }) } fn get_validator_registration(pubkey: PublicKeyBytes) -> ValidatorRegistrationData { @@ -771,28 +771,28 @@ mod tests { let first_attestation = || { let mut attestation = get_attestation(); - attestation.data.source.epoch = Epoch::new(1); - attestation.data.target.epoch = Epoch::new(4); + attestation.data_mut().source.epoch = Epoch::new(1); + attestation.data_mut().target.epoch = Epoch::new(4); attestation }; let double_vote_attestation = || { let mut attestation = first_attestation(); - attestation.data.beacon_block_root = Hash256::from_low_u64_be(1); + attestation.data_mut().beacon_block_root = Hash256::from_low_u64_be(1); attestation }; let surrounding_attestation = || { let mut attestation = first_attestation(); - attestation.data.source.epoch = Epoch::new(0); - attestation.data.target.epoch = Epoch::new(5); + attestation.data_mut().source.epoch = Epoch::new(0); + attestation.data_mut().target.epoch = Epoch::new(5); attestation }; let surrounded_attestation = || { let mut attestation = first_attestation(); - attestation.data.source.epoch = Epoch::new(2); - attestation.data.target.epoch = Epoch::new(3); + attestation.data_mut().source.epoch = Epoch::new(2); + attestation.data_mut().target.epoch = Epoch::new(3); attestation }; diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 1c6b60addb6..653d829d685 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -15,8 +15,8 @@ use std::sync::Arc; use tokio::time::{sleep, sleep_until, Duration, Instant}; use tree_hash::TreeHash; use types::{ - AggregateSignature, Attestation, AttestationData, BitList, ChainSpec, CommitteeIndex, EthSpec, - Slot, + attestation::AttestationBase, AggregateSignature, Attestation, AttestationData, BitList, + ChainSpec, CommitteeIndex, EthSpec, Slot, }; /// Builds an `AttestationService`. @@ -378,11 +378,11 @@ impl AttestationService { return None; } - let mut attestation = Attestation { + let mut attestation = Attestation::Base(AttestationBase { aggregation_bits: BitList::with_capacity(duty.committee_length as usize).unwrap(), data: attestation_data.clone(), signature: AggregateSignature::infinity(), - }; + }); match self .validator_store @@ -610,10 +610,10 @@ impl AttestationService { log, "Successfully published attestation"; "aggregator" => signed_aggregate_and_proof.message.aggregator_index, - "signatures" => attestation.aggregation_bits.num_set_bits(), - "head_block" => format!("{:?}", attestation.data.beacon_block_root), - "committee_index" => attestation.data.index, - "slot" => attestation.data.slot.as_u64(), + "signatures" => attestation.num_set_aggregation_bits(), + "head_block" => format!("{:?}", attestation.data().beacon_block_root), + "committee_index" => attestation.data().index, + "slot" => attestation.data().slot.as_u64(), "type" => "aggregated", ); } @@ -626,8 +626,8 @@ impl AttestationService { "Failed to publish attestation"; "error" => %e, "aggregator" => signed_aggregate_and_proof.message.aggregator_index, - "committee_index" => attestation.data.index, - "slot" => attestation.data.slot.as_u64(), + "committee_index" => attestation.data().index, + "slot" => attestation.data().slot.as_u64(), "type" => "aggregated", ); } diff --git a/validator_client/src/http_api/tests/keystores.rs b/validator_client/src/http_api/tests/keystores.rs index fe58393bb8d..b6923d1c788 100644 --- a/validator_client/src/http_api/tests/keystores.rs +++ b/validator_client/src/http_api/tests/keystores.rs @@ -13,7 +13,7 @@ use rand::{rngs::SmallRng, Rng, SeedableRng}; use slashing_protection::interchange::{Interchange, InterchangeMetadata}; use std::{collections::HashMap, path::Path}; use tokio::runtime::Handle; -use types::Address; +use types::{attestation::AttestationBase, Address}; fn new_keystore(password: ZeroizeString) -> Keystore { let keypair = Keypair::random(); @@ -1094,7 +1094,7 @@ async fn generic_migration_test( // Sign attestations on VC1. for (validator_index, mut attestation) in first_vc_attestations { let public_key = keystore_pubkey(&keystores[validator_index]); - let current_epoch = attestation.data.target.epoch; + let current_epoch = attestation.data().target.epoch; tester1 .validator_store .sign_attestation(public_key, 0, &mut attestation, current_epoch) @@ -1170,7 +1170,7 @@ async fn generic_migration_test( // Sign attestations on the second VC. for (validator_index, mut attestation, should_succeed) in second_vc_attestations { let public_key = keystore_pubkey(&keystores[validator_index]); - let current_epoch = attestation.data.target.epoch; + let current_epoch = attestation.data().target.epoch; match tester2 .validator_store .sign_attestation(public_key, 0, &mut attestation, current_epoch) @@ -1236,7 +1236,7 @@ async fn delete_nonexistent_keystores() { } fn make_attestation(source_epoch: u64, target_epoch: u64) -> Attestation { - Attestation { + Attestation::Base(AttestationBase { aggregation_bits: BitList::with_capacity( ::MaxValidatorsPerCommittee::to_usize(), ) @@ -1253,7 +1253,7 @@ fn make_attestation(source_epoch: u64, target_epoch: u64) -> Attestation { ..AttestationData::default() }, signature: AggregateSignature::empty(), - } + }) } #[tokio::test] diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 0a00dad9beb..7334c35bfc7 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -647,9 +647,9 @@ impl ValidatorStore { current_epoch: Epoch, ) -> Result<(), Error> { // Make sure the target epoch is not higher than the current epoch to avoid potential attacks. - if attestation.data.target.epoch > current_epoch { + if attestation.data().target.epoch > current_epoch { return Err(Error::GreaterThanCurrentEpoch { - epoch: attestation.data.target.epoch, + epoch: attestation.data().target.epoch, current_epoch, }); } @@ -658,7 +658,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; // Checking for slashing conditions. - let signing_epoch = attestation.data.target.epoch; + let signing_epoch = attestation.data().target.epoch; let signing_context = self.signing_context(Domain::BeaconAttester, signing_epoch); let domain_hash = signing_context.domain_hash(&self.spec); let slashing_status = if signing_method @@ -666,7 +666,7 @@ impl ValidatorStore { { self.slashing_protection.check_and_insert_attestation( &validator_pubkey, - &attestation.data, + attestation.data(), domain_hash, ) } else { @@ -678,7 +678,7 @@ impl ValidatorStore { Ok(Safe::Valid) => { let signature = signing_method .get_signature::>( - SignableMessage::AttestationData(&attestation.data), + SignableMessage::AttestationData(attestation.data()), signing_context, &self.spec, &self.task_executor, @@ -720,7 +720,7 @@ impl ValidatorStore { crit!( self.log, "Not signing slashable attestation"; - "attestation" => format!("{:?}", attestation.data), + "attestation" => format!("{:?}", attestation.data()), "error" => format!("{:?}", e) ); metrics::inc_counter_vec( @@ -798,7 +798,7 @@ impl ValidatorStore { aggregate: Attestation, selection_proof: SelectionProof, ) -> Result, Error> { - let signing_epoch = aggregate.data.target.epoch; + let signing_epoch = aggregate.data().target.epoch; let signing_context = self.signing_context(Domain::AggregateAndProof, signing_epoch); let message = AggregateAndProof { From e6c7f145dd1b8e1685cd9dd5fc766d5a716b6882 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Thu, 2 May 2024 18:00:21 -0500 Subject: [PATCH 002/227] `superstruct` the `AttesterSlashing` (#5636) * `superstruct` Attester Fork Variants * Push a little further * Deal with Encode / Decode of AttesterSlashing * not so sure about this.. * Stop Encode/Decode Bounds from Propagating Out * Tons of Changes.. * More Conversions to AttestationRef * Add AsReference trait (#15) * Add AsReference trait * Fix some snafus * Got it Compiling! :D * Got Tests Building * Get beacon chain tests compiling --------- Co-authored-by: Michael Sproul --- .../src/attestation_verification.rs | 8 +- beacon_node/beacon_chain/src/beacon_chain.rs | 107 +++++++++-- beacon_node/beacon_chain/src/block_reward.rs | 7 +- .../beacon_chain/src/block_verification.rs | 2 +- .../beacon_chain/src/observed_aggregates.rs | 80 +++++--- .../beacon_chain/src/observed_operations.rs | 9 +- beacon_node/beacon_chain/src/test_utils.rs | 52 +++--- .../beacon_chain/src/validator_monitor.rs | 23 +-- .../tests/attestation_production.rs | 47 +++-- .../tests/attestation_verification.rs | 150 +++++++++------ .../beacon_chain/tests/block_verification.rs | 167 +++++++++++++---- .../tests/payload_invalidation.rs | 20 +- beacon_node/beacon_chain/tests/store_tests.rs | 7 +- beacon_node/beacon_chain/tests/tests.rs | 2 +- .../http_api/src/block_packing_efficiency.rs | 10 +- beacon_node/http_api/src/lib.rs | 9 +- beacon_node/http_api/tests/tests.rs | 23 ++- .../lighthouse_network/src/types/pubsub.rs | 36 +++- .../gossip_methods.rs | 2 +- .../operation_pool/src/attestation_storage.rs | 19 +- .../operation_pool/src/attester_slashing.rs | 18 +- beacon_node/operation_pool/src/lib.rs | 35 ++-- beacon_node/operation_pool/src/persistence.rs | 4 +- consensus/fork_choice/src/fork_choice.rs | 20 +- .../src/common/get_attesting_indices.rs | 6 +- .../src/common/get_indexed_attestation.rs | 6 +- .../state_processing/src/consensus_context.rs | 18 +- consensus/state_processing/src/lib.rs | 2 +- .../block_signature_verifier.rs | 6 +- .../is_valid_indexed_attestation.rs | 2 +- .../process_operations.rs | 46 +++-- .../per_block_processing/signature_sets.rs | 18 +- .../src/per_block_processing/tests.rs | 104 ++++++++--- .../verify_attestation.rs | 10 +- .../verify_attester_slashing.rs | 14 +- .../state_processing/src/verify_operation.rs | 172 ++++++++++++++++- consensus/types/src/attestation.rs | 18 +- consensus/types/src/attester_slashing.rs | 175 ++++++++++++++++-- consensus/types/src/beacon_block.rs | 69 +++++-- consensus/types/src/beacon_block_body.rs | 138 +++++++++++++- consensus/types/src/indexed_attestation.rs | 60 +++++- consensus/types/src/lib.rs | 14 +- consensus/types/src/signed_beacon_block.rs | 2 + .../types/src/sync_committee_contribution.rs | 6 + lcli/src/indexed_attestations.rs | 2 +- lcli/src/transition_blocks.rs | 2 +- slasher/src/lib.rs | 49 ++++- slasher/src/slasher.rs | 2 +- slasher/src/test_utils.rs | 26 ++- testing/ef_tests/src/cases/fork_choice.rs | 10 +- testing/ef_tests/src/cases/operations.rs | 8 +- validator_client/src/block_service.rs | 4 +- watch/src/database/mod.rs | 2 +- 53 files changed, 1408 insertions(+), 440 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 131932febba..a95160d504d 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -471,7 +471,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { if chain .observed_attestations .write() - .is_known_subset(attestation, attestation_data_root) + .is_known_subset(attestation.to_ref(), attestation_data_root) .map_err(|e| Error::BeaconChainError(e.into()))? { metrics::inc_counter(&metrics::AGGREGATED_ATTESTATION_SUBSETS); @@ -559,7 +559,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { return Err(Error::AggregatorNotInCommittee { aggregator_index }); } - get_indexed_attestation(committee.committee, attestation) + get_indexed_attestation(committee.committee, attestation.to_ref()) .map_err(|e| BeaconChainError::from(e).into()) }; @@ -597,7 +597,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { if let ObserveOutcome::Subset = chain .observed_attestations .write() - .observe_item(attestation, Some(attestation_data_root)) + .observe_item(attestation.to_ref(), Some(attestation_data_root)) .map_err(|e| Error::BeaconChainError(e.into()))? { metrics::inc_counter(&metrics::AGGREGATED_ATTESTATION_SUBSETS); @@ -1241,7 +1241,7 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( attestation: &Attestation, ) -> Result<(IndexedAttestation, CommitteesPerSlot), Error> { map_attestation_committee(chain, attestation, |(committee, committees_per_slot)| { - get_indexed_attestation(committee.committee, attestation) + get_indexed_attestation(committee.committee, attestation.to_ref()) .map(|attestation| (attestation, committees_per_slot)) .map_err(Error::Invalid) }) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d8a546257be..ffd19dca0af 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2113,7 +2113,7 @@ impl BeaconChain { .fork_choice_write_lock() .on_attestation( self.slot()?, - verified.indexed_attestation(), + verified.indexed_attestation().to_ref(), AttestationFromBlock::False, ) .map_err(Into::into) @@ -2465,7 +2465,7 @@ impl BeaconChain { // Add to fork choice. self.canonical_head .fork_choice_write_lock() - .on_attester_slashing(attester_slashing.as_inner()); + .on_attester_slashing(attester_slashing.as_inner().to_ref()); // Add to the op pool (if we have the ability to propose blocks). if self.eth1_chain.is_some() { @@ -3820,7 +3820,7 @@ impl BeaconChain { continue; } }; - slasher.accept_attestation(indexed_attestation.clone()); + slasher.accept_attestation(indexed_attestation.clone_as_indexed_attestation()); } } } @@ -3839,7 +3839,7 @@ impl BeaconChain { if block.slot() + 2 * T::EthSpec::slots_per_epoch() >= current_slot { metrics::observe( &metrics::OPERATIONS_PER_BLOCK_ATTESTATION, - block.body().attestations().len() as f64, + block.body().attestations().count() as f64, ); if let Ok(sync_aggregate) = block.body().sync_aggregate() { @@ -4859,7 +4859,8 @@ impl BeaconChain { metrics::start_timer(&metrics::BLOCK_PRODUCTION_UNAGGREGATED_TIMES); for attestation in self.naive_aggregation_pool.read().iter() { let import = |attestation: &Attestation| { - let attesting_indices = get_attesting_indices_from_state(&state, attestation)?; + let attesting_indices = + get_attesting_indices_from_state(&state, attestation.to_ref())?; self.op_pool .insert_attestation(attestation.clone(), attesting_indices) }; @@ -4909,7 +4910,7 @@ impl BeaconChain { attestations.retain(|att| { verify_attestation_for_block_inclusion( &state, - att, + att.to_ref(), &mut tmp_ctxt, VerifySignatures::True, &self.spec, @@ -5040,6 +5041,72 @@ impl BeaconChain { bls_to_execution_changes, } = partial_beacon_block; + let (attester_slashings_base, attester_slashings_electra) = + attester_slashings.into_iter().fold( + (Vec::new(), Vec::new()), + |(mut base, mut electra), slashing| { + match slashing { + AttesterSlashing::Base(slashing) => base.push(slashing), + AttesterSlashing::Electra(slashing) => electra.push(slashing), + } + (base, electra) + }, + ); + let (attestations_base, attestations_electra) = attestations.into_iter().fold( + (Vec::new(), Vec::new()), + |(mut base, mut electra), attestation| { + match attestation { + Attestation::Base(attestation) => base.push(attestation), + Attestation::Electra(attestation) => electra.push(attestation), + } + (base, electra) + }, + ); + + // TODO(electra): figure out what should *actually* be done here when we have attestations / attester_slashings of the wrong type + match &state { + BeaconState::Base(_) + | BeaconState::Altair(_) + | BeaconState::Merge(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) => { + if !attestations_electra.is_empty() { + error!( + self.log, + "Tried to produce block with attestations of the wrong type"; + "slot" => slot, + "attestations" => attestations_electra.len(), + ); + } + if !attester_slashings_electra.is_empty() { + error!( + self.log, + "Tried to produce block with attester slashings of the wrong type"; + "slot" => slot, + "attester_slashings" => attester_slashings_electra.len(), + ); + } + } + BeaconState::Electra(_) => { + if !attestations_base.is_empty() { + error!( + self.log, + "Tried to produce block with attestations of the wrong type"; + "slot" => slot, + "attestations" => attestations_base.len(), + ); + } + if !attester_slashings_base.is_empty() { + error!( + self.log, + "Tried to produce block with attester slashings of the wrong type"; + "slot" => slot, + "attester_slashings" => attester_slashings_base.len(), + ); + } + } + }; + let (inner_block, maybe_blobs_and_proofs, execution_payload_value) = match &state { BeaconState::Base(_) => ( BeaconBlock::Base(BeaconBlockBase { @@ -5052,8 +5119,8 @@ impl BeaconChain { eth1_data, graffiti, proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), + attester_slashings: attester_slashings_base.into(), + attestations: attestations_base.into(), deposits: deposits.into(), voluntary_exits: voluntary_exits.into(), _phantom: PhantomData, @@ -5073,8 +5140,8 @@ impl BeaconChain { eth1_data, graffiti, proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), + attester_slashings: attester_slashings_base.into(), + attestations: attestations_base.into(), deposits: deposits.into(), voluntary_exits: voluntary_exits.into(), sync_aggregate: sync_aggregate @@ -5100,8 +5167,8 @@ impl BeaconChain { eth1_data, graffiti, proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), + attester_slashings: attester_slashings_base.into(), + attestations: attestations_base.into(), deposits: deposits.into(), voluntary_exits: voluntary_exits.into(), sync_aggregate: sync_aggregate @@ -5132,8 +5199,8 @@ impl BeaconChain { eth1_data, graffiti, proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), + attester_slashings: attester_slashings_base.into(), + attestations: attestations_base.into(), deposits: deposits.into(), voluntary_exits: voluntary_exits.into(), sync_aggregate: sync_aggregate @@ -5166,8 +5233,8 @@ impl BeaconChain { eth1_data, graffiti, proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), + attester_slashings: attester_slashings_base.into(), + attestations: attestations_base.into(), deposits: deposits.into(), voluntary_exits: voluntary_exits.into(), sync_aggregate: sync_aggregate @@ -5204,8 +5271,8 @@ impl BeaconChain { eth1_data, graffiti, proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), + attester_slashings: attester_slashings_electra.into(), + attestations: attestations_electra.into(), deposits: deposits.into(), voluntary_exits: voluntary_exits.into(), sync_aggregate: sync_aggregate @@ -5216,6 +5283,8 @@ impl BeaconChain { bls_to_execution_changes: bls_to_execution_changes.into(), blob_kzg_commitments: kzg_commitments .ok_or(BlockProductionError::InvalidPayloadFork)?, + // TODO(electra): finish consolidations when they're more spec'd out + consolidations: Vec::new().into(), }, }), maybe_blobs_and_proofs, @@ -5321,7 +5390,7 @@ impl BeaconChain { self.log, "Produced beacon block"; "parent" => ?block.parent_root(), - "attestations" => block.body().attestations().len(), + "attestations" => block.body().attestations_len(), "slot" => block.slot() ); diff --git a/beacon_node/beacon_chain/src/block_reward.rs b/beacon_node/beacon_chain/src/block_reward.rs index 44938668fe9..69eecc89b8a 100644 --- a/beacon_node/beacon_chain/src/block_reward.rs +++ b/beacon_node/beacon_chain/src/block_reward.rs @@ -27,10 +27,12 @@ impl BeaconChain { let split_attestations = block .body() .attestations() - .iter() .map(|att| { let attesting_indices = get_attesting_indices_from_state(state, att)?; - Ok(SplitAttestation::new(att.clone(), attesting_indices)) + Ok(SplitAttestation::new( + att.clone_as_attestation(), + attesting_indices, + )) }) .collect::, BeaconChainError>>()?; @@ -86,7 +88,6 @@ impl BeaconChain { block .body() .attestations() - .iter() .map(|a| a.data().clone()) .collect() } else { diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 866dde5a763..6536e6f4b30 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -1623,7 +1623,7 @@ impl ExecutionPendingBlock { } // Register each attestation in the block with fork choice. - for (i, attestation) in block.message().body().attestations().iter().enumerate() { + for (i, attestation) in block.message().body().attestations().enumerate() { let _fork_choice_attestation_timer = metrics::start_timer(&metrics::FORK_CHOICE_PROCESS_ATTESTATION_TIMES); diff --git a/beacon_node/beacon_chain/src/observed_aggregates.rs b/beacon_node/beacon_chain/src/observed_aggregates.rs index a83eb620788..857b0edb348 100644 --- a/beacon_node/beacon_chain/src/observed_aggregates.rs +++ b/beacon_node/beacon_chain/src/observed_aggregates.rs @@ -10,7 +10,7 @@ use types::consts::altair::{ SYNC_COMMITTEE_SUBNET_COUNT, TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE, }; use types::slot_data::SlotData; -use types::{Attestation, EthSpec, Hash256, Slot, SyncCommitteeContribution}; +use types::{Attestation, AttestationRef, EthSpec, Hash256, Slot, SyncCommitteeContribution}; pub type ObservedSyncContributions = ObservedAggregates< SyncCommitteeContribution, @@ -102,30 +102,30 @@ pub trait SubsetItem { fn root(&self) -> Hash256; } -impl SubsetItem for Attestation { +impl<'a, E: EthSpec> SubsetItem for AttestationRef<'a, E> { type Item = BitList; fn is_subset(&self, other: &Self::Item) -> bool { match self { - Attestation::Base(att) => att.aggregation_bits.is_subset(other), + Self::Base(att) => att.aggregation_bits.is_subset(other), // TODO(electra) implement electra variant - Attestation::Electra(_) => todo!(), + Self::Electra(_) => todo!(), } } fn is_superset(&self, other: &Self::Item) -> bool { match self { - Attestation::Base(att) => other.is_subset(&att.aggregation_bits), + Self::Base(att) => other.is_subset(&att.aggregation_bits), // TODO(electra) implement electra variant - Attestation::Electra(_) => todo!(), + Self::Electra(_) => todo!(), } } /// Returns the sync contribution aggregation bits. fn get_item(&self) -> Self::Item { match self { - Attestation::Base(att) => att.aggregation_bits.clone(), + Self::Base(att) => att.aggregation_bits.clone(), // TODO(electra) implement electra variant - Attestation::Electra(_) => todo!(), + Self::Electra(_) => todo!(), } } @@ -135,7 +135,7 @@ impl SubsetItem for Attestation { } } -impl SubsetItem for SyncCommitteeContribution { +impl<'a, E: EthSpec> SubsetItem for &'a SyncCommitteeContribution { type Item = BitVector; fn is_subset(&self, other: &Self::Item) -> bool { self.aggregation_bits.is_subset(other) @@ -208,7 +208,7 @@ impl SlotHashSet { /// Store the items in self so future observations recognise its existence. pub fn observe_item>( &mut self, - item: &S, + item: S, root: Hash256, ) -> Result { if item.get_slot() != self.slot { @@ -254,7 +254,7 @@ impl SlotHashSet { /// the given root and slot. pub fn is_known_subset>( &self, - item: &S, + item: S, root: Hash256, ) -> Result { if item.get_slot() != self.slot { @@ -276,16 +276,43 @@ impl SlotHashSet { } } +/// Trait for observable items that can be observed from their reference type. +/// +/// This is used to make observations for `Attestation`s from `AttestationRef`s. +pub trait AsReference { + type Reference<'a> + where + Self: 'a; + + fn as_reference(&self) -> Self::Reference<'_>; +} + +impl AsReference for Attestation { + type Reference<'a> = AttestationRef<'a, E>; + + fn as_reference(&self) -> AttestationRef<'_, E> { + self.to_ref() + } +} + +impl AsReference for SyncCommitteeContribution { + type Reference<'a> = &'a Self; + + fn as_reference(&self) -> &Self { + self + } +} + /// Stores the roots of objects for some number of `Slots`, so we can determine if /// these have previously been seen on the network. -pub struct ObservedAggregates { +pub struct ObservedAggregates { lowest_permissible_slot: Slot, sets: Vec>, _phantom_spec: PhantomData, _phantom_tree_hash: PhantomData, } -impl Default for ObservedAggregates { +impl Default for ObservedAggregates { fn default() -> Self { Self { lowest_permissible_slot: Slot::new(0), @@ -296,13 +323,18 @@ impl Default for ObservedAggregates, E: EthSpec, I> ObservedAggregates { +impl ObservedAggregates +where + T: Consts + AsReference, + E: EthSpec, + for<'a> T::Reference<'a>: SubsetItem + SlotData, +{ /// Store `item` in `self` keyed at `root`. /// /// `root` must equal `item.root::()`. pub fn observe_item( &mut self, - item: &T, + item: T::Reference<'_>, root_opt: Option, ) -> Result { let index = self.get_set_index(item.get_slot())?; @@ -319,7 +351,11 @@ impl, E: EthSpec, I> ObservedAggrega /// /// `root` must equal `item.root::()`. #[allow(clippy::wrong_self_convention)] - pub fn is_known_subset(&mut self, item: &T, root: Hash256) -> Result { + pub fn is_known_subset( + &mut self, + item: T::Reference<'_>, + root: Hash256, + ) -> Result { let index = self.get_set_index(item.get_slot())?; self.sets @@ -417,8 +453,8 @@ mod tests { fn get_attestation(slot: Slot, beacon_block_root: u64) -> Attestation { let mut a: Attestation = test_random_instance(); - a.data.slot = slot; - a.data.beacon_block_root = Hash256::from_low_u64_be(beacon_block_root); + a.data_mut().slot = slot; + a.data_mut().beacon_block_root = Hash256::from_low_u64_be(beacon_block_root); a } @@ -444,12 +480,12 @@ mod tests { for a in &items { assert_eq!( - store.is_known_subset(a, a.root()), + store.is_known_subset(a.as_reference(), a.as_reference().root()), Ok(false), "should indicate an unknown attestation is unknown" ); assert_eq!( - store.observe_item(a, None), + store.observe_item(a.as_reference(), None), Ok(ObserveOutcome::New), "should observe new attestation" ); @@ -457,12 +493,12 @@ mod tests { for a in &items { assert_eq!( - store.is_known_subset(a, a.root()), + store.is_known_subset(a.as_reference(), a.as_reference().root()), Ok(true), "should indicate a known attestation is known" ); assert_eq!( - store.observe_item(a, Some(a.root())), + store.observe_item(a.as_reference(), Some(a.as_reference().root())), Ok(ObserveOutcome::Subset), "should acknowledge an existing attestation" ); diff --git a/beacon_node/beacon_chain/src/observed_operations.rs b/beacon_node/beacon_chain/src/observed_operations.rs index a61cc4f74cc..969d03a11b6 100644 --- a/beacon_node/beacon_chain/src/observed_operations.rs +++ b/beacon_node/beacon_chain/src/observed_operations.rs @@ -1,7 +1,6 @@ use derivative::Derivative; use smallvec::{smallvec, SmallVec}; -use ssz::{Decode, Encode}; -use state_processing::{SigVerifiedOp, VerifyOperation, VerifyOperationAt}; +use state_processing::{SigVerifiedOp, TransformPersist, VerifyOperation, VerifyOperationAt}; use std::collections::HashSet; use std::marker::PhantomData; use types::{ @@ -34,7 +33,7 @@ pub struct ObservedOperations, E: EthSpec> { /// Was the observed operation new and valid for further processing, or a useless duplicate? #[derive(Debug, PartialEq, Eq, Clone)] -pub enum ObservationOutcome { +pub enum ObservationOutcome { New(SigVerifiedOp), AlreadyKnown, } @@ -62,12 +61,12 @@ impl ObservableOperation for ProposerSlashing { impl ObservableOperation for AttesterSlashing { fn observed_validators(&self) -> SmallVec<[u64; SMALL_VEC_SIZE]> { let attestation_1_indices = self - .attestation_1 + .attestation_1() .attesting_indices_iter() .copied() .collect::>(); let attestation_2_indices = self - .attestation_2 + .attestation_2() .attesting_indices_iter() .copied() .collect::>(); diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 4a53f64fcac..deefb34ed56 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1492,7 +1492,8 @@ where ) -> AttesterSlashing { let fork = self.chain.canonical_head.cached_head().head_fork(); - let mut attestation_1 = IndexedAttestation::Base(IndexedAttestationBase { + // TODO(electra): consider making this test fork-agnostic + let mut attestation_1 = IndexedAttestationBase { attesting_indices: VariableList::new(validator_indices).unwrap(), data: AttestationData { slot: Slot::new(0), @@ -1508,36 +1509,37 @@ where }, }, signature: AggregateSignature::infinity(), - }); + }; let mut attestation_2 = attestation_1.clone(); - attestation_2.data_mut().index += 1; - attestation_2.data_mut().source.epoch = source2.unwrap_or(Epoch::new(0)); - attestation_2.data_mut().target.epoch = target2.unwrap_or(fork.epoch); + attestation_2.data.index += 1; + attestation_2.data.source.epoch = source2.unwrap_or(Epoch::new(0)); + attestation_2.data.target.epoch = target2.unwrap_or(fork.epoch); for attestation in &mut [&mut attestation_1, &mut attestation_2] { // TODO(electra) we could explore iter mut here - for i in attestation.attesting_indices_to_vec() { - let sk = &self.validator_keypairs[i as usize].sk; + for i in attestation.attesting_indices.iter() { + let sk = &self.validator_keypairs[*i as usize].sk; let genesis_validators_root = self.chain.genesis_validators_root; let domain = self.chain.spec.get_domain( - attestation.data().target.epoch, + attestation.data.target.epoch, Domain::BeaconAttester, &fork, genesis_validators_root, ); - let message = attestation.data().signing_root(domain); + let message = attestation.data.signing_root(domain); - attestation.signature_mut().add_assign(&sk.sign(message)); + attestation.signature.add_assign(&sk.sign(message)); } } - AttesterSlashing { + // TODO(electra): fix this test + AttesterSlashing::Base(AttesterSlashingBase { attestation_1, attestation_2, - } + }) } pub fn make_attester_slashing_different_indices( @@ -1559,43 +1561,45 @@ where }, }; - let mut attestation_1 = IndexedAttestation::Base(IndexedAttestationBase { + // TODO(electra): make this test fork-agnostic + let mut attestation_1 = IndexedAttestationBase { attesting_indices: VariableList::new(validator_indices_1).unwrap(), data: data.clone(), signature: AggregateSignature::infinity(), - }); + }; - let mut attestation_2 = IndexedAttestation::Base(IndexedAttestationBase { + let mut attestation_2 = IndexedAttestationBase { attesting_indices: VariableList::new(validator_indices_2).unwrap(), data, signature: AggregateSignature::infinity(), - }); + }; - attestation_2.data_mut().index += 1; + attestation_2.data.index += 1; let fork = self.chain.canonical_head.cached_head().head_fork(); for attestation in &mut [&mut attestation_1, &mut attestation_2] { - for i in attestation.attesting_indices_to_vec() { - let sk = &self.validator_keypairs[i as usize].sk; + for i in attestation.attesting_indices.iter() { + let sk = &self.validator_keypairs[*i as usize].sk; let genesis_validators_root = self.chain.genesis_validators_root; let domain = self.chain.spec.get_domain( - attestation.data().target.epoch, + attestation.data.target.epoch, Domain::BeaconAttester, &fork, genesis_validators_root, ); - let message = attestation.data().signing_root(domain); + let message = attestation.data.signing_root(domain); - attestation.signature_mut().add_assign(&sk.sign(message)); + attestation.signature.add_assign(&sk.sign(message)); } } - AttesterSlashing { + // TODO(electra): fix this test + AttesterSlashing::Base(AttesterSlashingBase { attestation_1, attestation_2, - } + }) } pub fn make_proposer_slashing(&self, validator_index: u64) -> ProposerSlashing { diff --git a/beacon_node/beacon_chain/src/validator_monitor.rs b/beacon_node/beacon_chain/src/validator_monitor.rs index 70f86ac138e..ea0fe46caeb 100644 --- a/beacon_node/beacon_chain/src/validator_monitor.rs +++ b/beacon_node/beacon_chain/src/validator_monitor.rs @@ -25,9 +25,10 @@ use types::consts::altair::{ TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX, TIMELY_TARGET_FLAG_INDEX, }; use types::{ - Attestation, AttestationData, AttesterSlashing, BeaconBlockRef, BeaconState, BeaconStateError, - ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, ProposerSlashing, PublicKeyBytes, - SignedAggregateAndProof, SignedContributionAndProof, Slot, SyncCommitteeMessage, VoluntaryExit, + Attestation, AttestationData, AttesterSlashingRef, BeaconBlockRef, BeaconState, + BeaconStateError, ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, + IndexedAttestationRef, ProposerSlashing, PublicKeyBytes, SignedAggregateAndProof, + SignedContributionAndProof, Slot, SyncCommitteeMessage, VoluntaryExit, }; /// Used for Prometheus labels. @@ -1410,7 +1411,7 @@ impl ValidatorMonitor { /// Note: Blocks that get orphaned will skew the inclusion distance calculation. pub fn register_attestation_in_block( &self, - indexed_attestation: &IndexedAttestation, + indexed_attestation: IndexedAttestationRef<'_, E>, parent_slot: Slot, spec: &ChainSpec, ) { @@ -1783,30 +1784,30 @@ impl ValidatorMonitor { } /// Register an attester slashing from the gossip network. - pub fn register_gossip_attester_slashing(&self, slashing: &AttesterSlashing) { + pub fn register_gossip_attester_slashing(&self, slashing: AttesterSlashingRef<'_, E>) { self.register_attester_slashing("gossip", slashing) } /// Register an attester slashing from the HTTP API. - pub fn register_api_attester_slashing(&self, slashing: &AttesterSlashing) { + pub fn register_api_attester_slashing(&self, slashing: AttesterSlashingRef<'_, E>) { self.register_attester_slashing("api", slashing) } /// Register an attester slashing included in a *valid* `BeaconBlock`. - pub fn register_block_attester_slashing(&self, slashing: &AttesterSlashing) { + pub fn register_block_attester_slashing(&self, slashing: AttesterSlashingRef<'_, E>) { self.register_attester_slashing("block", slashing) } - fn register_attester_slashing(&self, src: &str, slashing: &AttesterSlashing) { - let data = slashing.attestation_1.data(); + fn register_attester_slashing(&self, src: &str, slashing: AttesterSlashingRef<'_, E>) { + let data = slashing.attestation_1().data(); let attestation_1_indices: HashSet = slashing - .attestation_1 + .attestation_1() .attesting_indices_iter() .copied() .collect(); slashing - .attestation_2 + .attestation_2() .attesting_indices_iter() .filter(|index| attestation_1_indices.contains(index)) .filter_map(|index| self.get_validator(*index)) diff --git a/beacon_node/beacon_chain/tests/attestation_production.rs b/beacon_node/beacon_chain/tests/attestation_production.rs index ff83b253205..a58cfd9f2d1 100644 --- a/beacon_node/beacon_chain/tests/attestation_production.rs +++ b/beacon_node/beacon_chain/tests/attestation_production.rs @@ -8,7 +8,9 @@ use beacon_chain::{metrics, StateSkipConfig, WhenSlotSkipped}; use lazy_static::lazy_static; use std::sync::Arc; use tree_hash::TreeHash; -use types::{AggregateSignature, EthSpec, Keypair, MainnetEthSpec, RelativeEpoch, Slot}; +use types::{ + AggregateSignature, Attestation, EthSpec, Keypair, MainnetEthSpec, RelativeEpoch, Slot, +}; pub const VALIDATOR_COUNT: usize = 16; @@ -188,20 +190,35 @@ async fn produces_attestations() { .produce_unaggregated_attestation(slot, index) .expect("should produce attestation"); - let data = &attestation.data; + match &attestation { + Attestation::Base(att) => { + assert_eq!( + att.aggregation_bits.len(), + committee_len, + "bad committee len" + ); + assert!( + att.aggregation_bits.is_zero(), + "some committee bits are set" + ); + } + Attestation::Electra(att) => { + assert_eq!( + att.aggregation_bits.len(), + committee_len, + "bad committee len" + ); + assert!( + att.aggregation_bits.is_zero(), + "some committee bits are set" + ); + } + } + let data = attestation.data(); assert_eq!( - attestation.aggregation_bits.len(), - committee_len, - "bad committee len" - ); - assert!( - attestation.aggregation_bits.is_zero(), - "some committee bits are set" - ); - assert_eq!( - attestation.signature, - AggregateSignature::empty(), + attestation.signature(), + &AggregateSignature::empty(), "bad signature" ); assert_eq!(data.index, index, "bad index"); @@ -329,10 +346,10 @@ async fn early_attester_cache_old_request() { .produce_unaggregated_attestation(attest_slot, 0) .unwrap(); - assert_eq!(attestation.data.slot, attest_slot); + assert_eq!(attestation.data().slot, attest_slot); let attested_block = harness .chain - .get_blinded_block(&attestation.data.beacon_block_root) + .get_blinded_block(&attestation.data().beacon_block_root) .unwrap() .unwrap(); assert_eq!(attested_block.slot(), attest_slot); diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index 3432604cc93..02caf831fe4 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -125,7 +125,7 @@ fn get_valid_unaggregated_attestation( let validator_committee_index = 0; let validator_index = *head .beacon_state - .get_beacon_committee(current_slot, valid_attestation.data.index) + .get_beacon_committee(current_slot, valid_attestation.data().index) .expect("should get committees") .committee .get(validator_committee_index) @@ -144,7 +144,7 @@ fn get_valid_unaggregated_attestation( .expect("should sign attestation"); let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - &valid_attestation.data, + valid_attestation.data(), head.beacon_state .get_committee_count_at_slot(current_slot) .expect("should get committee count"), @@ -170,7 +170,7 @@ fn get_valid_aggregated_attestation( let current_slot = chain.slot().expect("should get slot"); let committee = state - .get_beacon_committee(current_slot, aggregate.data.index) + .get_beacon_committee(current_slot, aggregate.data().index) .expect("should get committees"); let committee_len = committee.committee.len(); @@ -181,7 +181,7 @@ fn get_valid_aggregated_attestation( let aggregator_sk = generate_deterministic_keypair(val_index).sk; let proof = SelectionProof::new::( - aggregate.data.slot, + aggregate.data().slot, &aggregator_sk, &state.fork(), chain.genesis_validators_root, @@ -220,7 +220,7 @@ fn get_non_aggregator( let current_slot = chain.slot().expect("should get slot"); let committee = state - .get_beacon_committee(current_slot, aggregate.data.index) + .get_beacon_committee(current_slot, aggregate.data().index) .expect("should get committees"); let committee_len = committee.committee.len(); @@ -231,7 +231,7 @@ fn get_non_aggregator( let aggregator_sk = generate_deterministic_keypair(val_index).sk; let proof = SelectionProof::new::( - aggregate.data.slot, + aggregate.data().slot, &aggregator_sk, &state.fork(), chain.genesis_validators_root, @@ -301,7 +301,7 @@ impl GossipTester { get_valid_aggregated_attestation(&harness.chain, valid_attestation.clone()); let mut invalid_attestation = valid_attestation.clone(); - invalid_attestation.data.beacon_block_root = Hash256::repeat_byte(13); + invalid_attestation.data_mut().beacon_block_root = Hash256::repeat_byte(13); let (mut invalid_aggregate, _, _) = get_valid_aggregated_attestation(&harness.chain, invalid_attestation.clone()); @@ -490,7 +490,7 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "aggregate from future slot", - |tester, a| a.message.aggregate.data.slot = tester.slot() + 1, + |tester, a| a.message.aggregate.data_mut().slot = tester.slot() + 1, |tester, err| { assert!(matches!( err, @@ -504,8 +504,9 @@ async fn aggregated_gossip_verification() { "aggregate from past slot", |tester, a| { let too_early_slot = tester.earliest_valid_attestation_slot() - 1; - a.message.aggregate.data.slot = too_early_slot; - a.message.aggregate.data.target.epoch = too_early_slot.epoch(E::slots_per_epoch()); + a.message.aggregate.data_mut().slot = too_early_slot; + a.message.aggregate.data_mut().target.epoch = + too_early_slot.epoch(E::slots_per_epoch()); }, |tester, err| { let valid_early_slot = tester.earliest_valid_attestation_slot(); @@ -529,7 +530,7 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "attestation with invalid target epoch", - |_, a| a.message.aggregate.data.target.epoch += 1, + |_, a| a.message.aggregate.data_mut().target.epoch += 1, |_, err| assert!(matches!(err, AttnError::InvalidTargetEpoch { .. })), ) /* @@ -538,7 +539,7 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "attestation with invalid target root", - |_, a| a.message.aggregate.data.target.root = Hash256::repeat_byte(42), + |_, a| a.message.aggregate.data_mut().target.root = Hash256::repeat_byte(42), |_, err| assert!(matches!(err, AttnError::InvalidTargetRoot { .. })), ) /* @@ -548,7 +549,7 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "aggregate with unknown head block", - |_, a| a.message.aggregate.data.beacon_block_root = Hash256::repeat_byte(42), + |_, a| a.message.aggregate.data_mut().beacon_block_root = Hash256::repeat_byte(42), |_, err| { assert!(matches!( err, @@ -567,10 +568,19 @@ async fn aggregated_gossip_verification() { .inspect_aggregate_err( "aggregate with no participants", |_, a| { - let aggregation_bits = &mut a.message.aggregate.aggregation_bits; - aggregation_bits.difference_inplace(&aggregation_bits.clone()); - assert!(aggregation_bits.is_zero()); - a.message.aggregate.signature = AggregateSignature::infinity(); + match &mut a.message.aggregate { + Attestation::Base(ref mut att) => { + let aggregation_bits = &mut att.aggregation_bits; + aggregation_bits.difference_inplace(&aggregation_bits.clone()); + assert!(aggregation_bits.is_zero()); + } + Attestation::Electra(ref mut att) => { + let aggregation_bits = &mut att.aggregation_bits; + aggregation_bits.difference_inplace(&aggregation_bits.clone()); + assert!(aggregation_bits.is_zero()); + } + } + *a.message.aggregate.signature_mut() = AggregateSignature::infinity(); }, |_, err| assert!(matches!(err, AttnError::EmptyAggregationBitfield)), ) @@ -598,7 +608,7 @@ async fn aggregated_gossip_verification() { .chain .head_snapshot() .beacon_state - .get_beacon_committee(tester.slot(), a.message.aggregate.data.index) + .get_beacon_committee(tester.slot(), a.message.aggregate.data().index) .expect("should get committees") .committee .len(); @@ -634,7 +644,7 @@ async fn aggregated_gossip_verification() { |tester, a| { let mut agg_sig = AggregateSignature::infinity(); agg_sig.add_assign(&tester.aggregator_sk.sign(Hash256::repeat_byte(42))); - a.message.aggregate.signature = agg_sig; + *a.message.aggregate.signature_mut() = agg_sig; }, |_, err| assert!(matches!(err, AttnError::InvalidSignature)), ) @@ -729,7 +739,7 @@ async fn aggregated_gossip_verification() { assert!(matches!( err, AttnError::AttestationSupersetKnown(hash) - if hash == tester.valid_aggregate.message.aggregate.data.tree_hash_root() + if hash == tester.valid_aggregate.message.aggregate.data().tree_hash_root() )) }, ) @@ -741,7 +751,7 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "aggregate from aggregator that has already been seen", - |_, a| a.message.aggregate.data.beacon_block_root = Hash256::repeat_byte(42), + |_, a| a.message.aggregate.data_mut().beacon_block_root = Hash256::repeat_byte(42), |tester, err| { assert!(matches!( err, @@ -766,12 +776,12 @@ async fn unaggregated_gossip_verification() { .inspect_unaggregate_err( "attestation with invalid committee index", |tester, a, _| { - a.data.index = tester + a.data_mut().index = tester .harness .chain .head_snapshot() .beacon_state - .get_committee_count_at_slot(a.data.slot) + .get_committee_count_at_slot(a.data().slot) .unwrap() }, |_, err| assert!(matches!(err, AttnError::NoCommitteeForSlotAndIndex { .. })), @@ -806,7 +816,7 @@ async fn unaggregated_gossip_verification() { */ .inspect_unaggregate_err( "attestation from future slot", - |tester, a, _| a.data.slot = tester.slot() + 1, + |tester, a, _| a.data_mut().slot = tester.slot() + 1, |tester, err| { assert!(matches!( err, @@ -822,8 +832,8 @@ async fn unaggregated_gossip_verification() { "attestation from past slot", |tester, a, _| { let too_early_slot = tester.earliest_valid_attestation_slot() - 1; - a.data.slot = too_early_slot; - a.data.target.epoch = too_early_slot.epoch(E::slots_per_epoch()); + a.data_mut().slot = too_early_slot; + a.data_mut().target.epoch = too_early_slot.epoch(E::slots_per_epoch()); }, |tester, err| { let valid_early_slot = tester.earliest_valid_attestation_slot(); @@ -847,7 +857,7 @@ async fn unaggregated_gossip_verification() { */ .inspect_unaggregate_err( "attestation with invalid target epoch", - |_, a, _| a.data.target.epoch += 1, + |_, a, _| a.data_mut().target.epoch += 1, |_, err| { assert!(matches!( err, @@ -863,15 +873,29 @@ async fn unaggregated_gossip_verification() { */ .inspect_unaggregate_err( "attestation without any aggregation bits set", - |tester, a, _| { - a.aggregation_bits - .set(tester.attester_committee_index, false) - .expect("should unset aggregation bit"); - assert_eq!( - a.aggregation_bits.num_set_bits(), - 0, - "test requires no set bits" - ); + |tester, mut a, _| { + match &mut a { + Attestation::Base(ref mut att) => { + att.aggregation_bits + .set(tester.attester_committee_index, false) + .expect("should unset aggregation bit"); + assert_eq!( + att.aggregation_bits.num_set_bits(), + 0, + "test requires no set bits" + ); + } + Attestation::Electra(ref mut att) => { + att.aggregation_bits + .set(tester.attester_committee_index, false) + .expect("should unset aggregation bit"); + assert_eq!( + att.aggregation_bits.num_set_bits(), + 0, + "test requires no set bits" + ); + } + } }, |_, err| { assert!(matches!( @@ -882,10 +906,19 @@ async fn unaggregated_gossip_verification() { ) .inspect_unaggregate_err( "attestation with two aggregation bits set", - |tester, a, _| { - a.aggregation_bits - .set(tester.attester_committee_index + 1, true) - .expect("should set second aggregation bit"); + |tester, mut a, _| { + match &mut a { + Attestation::Base(ref mut att) => { + att.aggregation_bits + .set(tester.attester_committee_index + 1, true) + .expect("should set second aggregation bit"); + } + Attestation::Electra(ref mut att) => { + att.aggregation_bits + .set(tester.attester_committee_index + 1, true) + .expect("should set second aggregation bit"); + } + } }, |_, err| { assert!(matches!( @@ -903,11 +936,22 @@ async fn unaggregated_gossip_verification() { */ .inspect_unaggregate_err( "attestation with invalid bitfield", - |_, a, _| { - let bits = a.aggregation_bits.iter().collect::>(); - a.aggregation_bits = BitList::with_capacity(bits.len() + 1).unwrap(); - for (i, bit) in bits.into_iter().enumerate() { - a.aggregation_bits.set(i, bit).unwrap(); + |_, mut a, _| { + match &mut a { + Attestation::Base(ref mut att) => { + let bits = att.aggregation_bits.iter().collect::>(); + att.aggregation_bits = BitList::with_capacity(bits.len() + 1).unwrap(); + for (i, bit) in bits.into_iter().enumerate() { + att.aggregation_bits.set(i, bit).unwrap(); + } + } + Attestation::Electra(ref mut att) => { + let bits = att.aggregation_bits.iter().collect::>(); + att.aggregation_bits = BitList::with_capacity(bits.len() + 1).unwrap(); + for (i, bit) in bits.into_iter().enumerate() { + att.aggregation_bits.set(i, bit).unwrap(); + } + } } }, |_, err| { @@ -927,7 +971,7 @@ async fn unaggregated_gossip_verification() { .inspect_unaggregate_err( "attestation with unknown head block", |_, a, _| { - a.data.beacon_block_root = Hash256::repeat_byte(42); + a.data_mut().beacon_block_root = Hash256::repeat_byte(42); }, |_, err| { assert!(matches!( @@ -949,7 +993,7 @@ async fn unaggregated_gossip_verification() { .inspect_unaggregate_err( "attestation with invalid target root", |_, a, _| { - a.data.target.root = Hash256::repeat_byte(42); + a.data_mut().target.root = Hash256::repeat_byte(42); }, |_, err| { assert!(matches!( @@ -968,7 +1012,7 @@ async fn unaggregated_gossip_verification() { |tester, a, _| { let mut agg_sig = AggregateSignature::infinity(); agg_sig.add_assign(&tester.attester_sk.sign(Hash256::repeat_byte(42))); - a.signature = agg_sig; + *a.signature_mut() = agg_sig; }, |_, err| { assert!(matches!( @@ -1055,7 +1099,7 @@ async fn attestation_that_skips_epochs() { .cloned() .expect("should have at least one attestation in committee"); - let block_root = attestation.data.beacon_block_root; + let block_root = attestation.data().beacon_block_root; let block_slot = harness .chain .store @@ -1066,7 +1110,7 @@ async fn attestation_that_skips_epochs() { .slot(); assert!( - attestation.data.slot - block_slot > E::slots_per_epoch() * 2, + attestation.data().slot - block_slot > E::slots_per_epoch() * 2, "the attestation must skip more than two epochs" ); @@ -1228,7 +1272,7 @@ async fn attestation_to_finalized_block() { .first() .cloned() .expect("should have at least one attestation in committee"); - assert_eq!(attestation.data.beacon_block_root, earlier_block_root); + assert_eq!(attestation.data().beacon_block_root, earlier_block_root); // Attestation should be rejected for attesting to a pre-finalization block. let res = harness @@ -1281,7 +1325,7 @@ async fn verify_aggregate_for_gossip_doppelganger_detection() { .verify_aggregated_attestation_for_gossip(&valid_aggregate) .expect("should verify aggregate attestation"); - let epoch = valid_aggregate.message.aggregate.data.target.epoch; + let epoch = valid_aggregate.message.aggregate.data().target.epoch; let index = valid_aggregate.message.aggregator_index as usize; assert!(harness.chain.validator_seen_at_epoch(index, epoch)); @@ -1338,7 +1382,7 @@ async fn verify_attestation_for_gossip_doppelganger_detection() { .verify_unaggregated_attestation_for_gossip(&valid_attestation, Some(subnet_id)) .expect("should verify attestation"); - let epoch = valid_attestation.data.target.epoch; + let epoch = valid_attestation.data().target.epoch; assert!(harness.chain.validator_seen_at_epoch(index, epoch)); // Check the correct beacon cache is populated diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 98a112daffe..7b2c1addd28 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -664,7 +664,7 @@ async fn invalid_signature_attester_slashing() { for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; let mut snapshots = chain_segment.clone(); - let indexed_attestation = IndexedAttestation { + let indexed_attestation = IndexedAttestationBase { attesting_indices: vec![0].into(), data: AttestationData { slot: Slot::new(0), @@ -681,7 +681,7 @@ async fn invalid_signature_attester_slashing() { }, signature: junk_aggregate_signature(), }; - let attester_slashing = AttesterSlashing { + let attester_slashing = AttesterSlashingBase { attestation_1: indexed_attestation.clone(), attestation_2: indexed_attestation, }; @@ -690,11 +690,36 @@ async fn invalid_signature_attester_slashing() { .as_ref() .clone() .deconstruct(); - block - .body_mut() - .attester_slashings_mut() - .push(attester_slashing) - .expect("should update attester slashing"); + match &mut block.body_mut() { + BeaconBlockBodyRefMut::Base(ref mut blk) => { + blk.attester_slashings + .push(attester_slashing) + .expect("should update attester slashing"); + } + BeaconBlockBodyRefMut::Altair(ref mut blk) => { + blk.attester_slashings + .push(attester_slashing) + .expect("should update attester slashing"); + } + BeaconBlockBodyRefMut::Merge(ref mut blk) => { + blk.attester_slashings + .push(attester_slashing) + .expect("should update attester slashing"); + } + BeaconBlockBodyRefMut::Capella(ref mut blk) => { + blk.attester_slashings + .push(attester_slashing) + .expect("should update attester slashing"); + } + BeaconBlockBodyRefMut::Deneb(ref mut blk) => { + blk.attester_slashings + .push(attester_slashing) + .expect("should update attester slashing"); + } + BeaconBlockBodyRefMut::Electra(_) => { + panic!("electra test not implemented!"); + } + } snapshots[block_index].beacon_block = Arc::new(SignedBeaconBlock::from_block(block, signature)); update_parent_roots(&mut snapshots, &mut chain_segment_blobs); @@ -724,8 +749,34 @@ async fn invalid_signature_attestation() { .as_ref() .clone() .deconstruct(); - if let Some(attestation) = block.body_mut().attestations_mut().get_mut(0) { - attestation.signature = junk_aggregate_signature(); + match &mut block.body_mut() { + BeaconBlockBodyRefMut::Base(ref mut blk) => blk + .attestations + .get_mut(0) + .map(|att| att.signature = junk_aggregate_signature()), + BeaconBlockBodyRefMut::Altair(ref mut blk) => blk + .attestations + .get_mut(0) + .map(|att| att.signature = junk_aggregate_signature()), + BeaconBlockBodyRefMut::Merge(ref mut blk) => blk + .attestations + .get_mut(0) + .map(|att| att.signature = junk_aggregate_signature()), + BeaconBlockBodyRefMut::Capella(ref mut blk) => blk + .attestations + .get_mut(0) + .map(|att| att.signature = junk_aggregate_signature()), + BeaconBlockBodyRefMut::Deneb(ref mut blk) => blk + .attestations + .get_mut(0) + .map(|att| att.signature = junk_aggregate_signature()), + BeaconBlockBodyRefMut::Electra(ref mut blk) => blk + .attestations + .get_mut(0) + .map(|att| att.signature = junk_aggregate_signature()), + }; + + if block.body().attestations_len() > 0 { snapshots[block_index].beacon_block = Arc::new(SignedBeaconBlock::from_block(block, signature)); update_parent_roots(&mut snapshots, &mut chain_segment_blobs); @@ -1187,9 +1238,13 @@ async fn verify_block_for_gossip_doppelganger_detection() { let state = harness.get_current_state(); let ((block, _), _) = harness.make_block(state.clone(), Slot::new(1)).await; - + let attestations = block + .message() + .body() + .attestations() + .map(|att| att.clone_as_attestation()) + .collect::>(); let verified_block = harness.chain.verify_block_for_gossip(block).await.unwrap(); - let attestations = verified_block.block.message().body().attestations().clone(); harness .chain .process_block( @@ -1202,37 +1257,69 @@ async fn verify_block_for_gossip_doppelganger_detection() { .unwrap(); for att in attestations.iter() { - let epoch = att.data.target.epoch; + let epoch = att.data().target.epoch; let committee = state - .get_beacon_committee(att.data.slot, att.data.index) + .get_beacon_committee(att.data().slot, att.data().index) .unwrap(); - let indexed_attestation = get_indexed_attestation(committee.committee, att).unwrap(); - - for &index in &indexed_attestation.attesting_indices { - let index = index as usize; - - assert!(harness.chain.validator_seen_at_epoch(index, epoch)); - - // Check the correct beacon cache is populated - assert!(harness - .chain - .observed_block_attesters - .read() - .validator_has_been_observed(epoch, index) - .expect("should check if block attester was observed")); - assert!(!harness - .chain - .observed_gossip_attesters - .read() - .validator_has_been_observed(epoch, index) - .expect("should check if gossip attester was observed")); - assert!(!harness - .chain - .observed_aggregators - .read() - .validator_has_been_observed(epoch, index) - .expect("should check if gossip aggregator was observed")); - } + let indexed_attestation = + get_indexed_attestation(committee.committee, att.to_ref()).unwrap(); + + match indexed_attestation { + IndexedAttestation::Base(indexed_attestation) => { + for &index in &indexed_attestation.attesting_indices { + let index = index as usize; + + assert!(harness.chain.validator_seen_at_epoch(index, epoch)); + + // Check the correct beacon cache is populated + assert!(harness + .chain + .observed_block_attesters + .read() + .validator_has_been_observed(epoch, index) + .expect("should check if block attester was observed")); + assert!(!harness + .chain + .observed_gossip_attesters + .read() + .validator_has_been_observed(epoch, index) + .expect("should check if gossip attester was observed")); + assert!(!harness + .chain + .observed_aggregators + .read() + .validator_has_been_observed(epoch, index) + .expect("should check if gossip aggregator was observed")); + } + } + IndexedAttestation::Electra(indexed_attestation) => { + for &index in &indexed_attestation.attesting_indices { + let index = index as usize; + + assert!(harness.chain.validator_seen_at_epoch(index, epoch)); + + // Check the correct beacon cache is populated + assert!(harness + .chain + .observed_block_attesters + .read() + .validator_has_been_observed(epoch, index) + .expect("should check if block attester was observed")); + assert!(!harness + .chain + .observed_gossip_attesters + .read() + .validator_has_been_observed(epoch, index) + .expect("should check if gossip attester was observed")); + assert!(!harness + .chain + .observed_aggregators + .read() + .validator_has_been_observed(epoch, index) + .expect("should check if gossip aggregator was observed")); + } + } + }; } } diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 0ef348319af..c1c770cfe4e 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -1191,9 +1191,17 @@ async fn attesting_to_optimistic_head() { .produce_unaggregated_attestation(Slot::new(0), 0) .unwrap(); - attestation.aggregation_bits.set(0, true).unwrap(); - attestation.data.slot = slot; - attestation.data.beacon_block_root = root; + match &mut attestation { + Attestation::Base(ref mut att) => { + att.aggregation_bits.set(0, true).unwrap(); + } + Attestation::Electra(ref mut att) => { + att.aggregation_bits.set(0, true).unwrap(); + } + } + + attestation.data_mut().slot = slot; + attestation.data_mut().beacon_block_root = root; rig.harness .chain @@ -1214,15 +1222,15 @@ async fn attesting_to_optimistic_head() { let get_aggregated = || { rig.harness .chain - .get_aggregated_attestation(&attestation.data) + .get_aggregated_attestation(attestation.data()) }; let get_aggregated_by_slot_and_root = || { rig.harness .chain .get_aggregated_attestation_by_slot_and_root( - attestation.data.slot, - &attestation.data.tree_hash_root(), + attestation.data().slot, + &attestation.data().tree_hash_root(), ) }; diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index ba8a6bf7016..8c0464187f3 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -606,7 +606,7 @@ async fn epoch_boundary_state_attestation_processing() { for (attestation, subnet_id) in late_attestations.into_iter().flatten() { // load_epoch_boundary_state is idempotent! - let block_root = attestation.data.beacon_block_root; + let block_root = attestation.data().beacon_block_root; let block = store .get_blinded_block(&block_root) .unwrap() @@ -629,7 +629,7 @@ async fn epoch_boundary_state_attestation_processing() { .verify_unaggregated_attestation_for_gossip(&attestation, Some(subnet_id)); let current_slot = harness.chain.slot().expect("should get slot"); - let expected_attestation_slot = attestation.data.slot; + let expected_attestation_slot = attestation.data().slot; // Extra -1 to handle gossip clock disparity. let expected_earliest_permissible_slot = current_slot - E::slots_per_epoch() - 1; @@ -1028,8 +1028,7 @@ async fn multiple_attestations_per_block() { .as_ref() .message() .body() - .attestations() - .len() as u64, + .attestations_len() as u64, if slot <= 1 { 0 } else { committees_per_slot } ); } diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index e27180a002c..3219611155e 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -573,7 +573,7 @@ async fn attestations_with_increasing_slots() { .verify_unaggregated_attestation_for_gossip(&attestation, Some(subnet_id)); let current_slot = harness.chain.slot().expect("should get slot"); - let expected_attestation_slot = attestation.data.slot; + let expected_attestation_slot = attestation.data().slot; let expected_earliest_permissible_slot = current_slot - MinimalEthSpec::slots_per_epoch() - 1; diff --git a/beacon_node/http_api/src/block_packing_efficiency.rs b/beacon_node/http_api/src/block_packing_efficiency.rs index 3e511c25dff..43dee5b6eb4 100644 --- a/beacon_node/http_api/src/block_packing_efficiency.rs +++ b/beacon_node/http_api/src/block_packing_efficiency.rs @@ -10,8 +10,8 @@ use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::sync::Arc; use types::{ - Attestation, BeaconCommittee, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, Epoch, - EthSpec, Hash256, OwnedBeaconCommittee, RelativeEpoch, SignedBeaconBlock, Slot, + AttestationRef, BeaconCommittee, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, + Epoch, EthSpec, Hash256, OwnedBeaconCommittee, RelativeEpoch, SignedBeaconBlock, Slot, }; use warp_utils::reject::{beacon_chain_error, custom_bad_request, custom_server_error}; @@ -111,9 +111,9 @@ impl PackingEfficiencyHandler { let attestations = block_body.attestations(); let mut attestations_in_block = HashMap::new(); - for attestation in attestations.iter() { + for attestation in attestations { match attestation { - Attestation::Base(attn) => { + AttestationRef::Base(attn) => { for (position, voted) in attn.aggregation_bits.iter().enumerate() { if voted { let unique_attestation = UniqueAttestation { @@ -133,7 +133,7 @@ impl PackingEfficiencyHandler { } } // TODO(electra) implement electra variant - Attestation::Electra(_) => { + AttestationRef::Electra(_) => { todo!() } } diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 87f7df07310..cc680c523ef 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1635,7 +1635,12 @@ pub fn serve( let (block, execution_optimistic, finalized) = block_id.blinded_block(&chain)?; Ok(api_types::GenericResponse::from( - block.message().body().attestations().clone(), + block + .message() + .body() + .attestations() + .map(|att| att.clone_as_attestation()) + .collect::>(), ) .add_execution_optimistic_finalized(execution_optimistic, finalized)) }) @@ -1833,7 +1838,7 @@ pub fn serve( chain .validator_monitor .read() - .register_api_attester_slashing(&slashing); + .register_api_attester_slashing(slashing.to_ref()); if let ObservationOutcome::New(slashing) = outcome { publish_pubsub_message( diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 21d93f22f33..87d6a8405d1 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -1633,7 +1633,13 @@ impl ApiTester { let expected = block_id.full_block(&self.chain).await.ok().map( |(block, _execution_optimistic, _finalized)| { - block.message().body().attestations().clone().into() + block + .message() + .body() + .attestations() + .map(|att| att.clone_as_attestation()) + .collect::>() + .into() }, ); @@ -1793,7 +1799,14 @@ impl ApiTester { pub async fn test_post_beacon_pool_attester_slashings_invalid(mut self) -> Self { let mut slashing = self.attester_slashing.clone(); - slashing.attestation_1.data_mut().slot += 1; + match &mut slashing { + AttesterSlashing::Base(ref mut slashing) => { + slashing.attestation_1.data.slot += 1; + } + AttesterSlashing::Electra(ref mut slashing) => { + slashing.attestation_1.data.slot += 1; + } + } self.client .post_beacon_pool_attester_slashings(&slashing) @@ -3183,8 +3196,10 @@ impl ApiTester { .head_beacon_block() .message() .body() - .attestations()[0] - .clone(); + .attestations() + .next() + .unwrap() + .clone_as_attestation(); let result = self .client diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 653e264349e..68864c76638 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -7,12 +7,12 @@ use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; use types::{ - Attestation, AttesterSlashing, BlobSidecar, EthSpec, ForkContext, ForkName, - LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, - SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, - SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, - SignedBeaconBlockMerge, SignedBlsToExecutionChange, SignedContributionAndProof, - SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, + Attestation, AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, + EthSpec, ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, + ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, + SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, + SignedBeaconBlockElectra, SignedBeaconBlockMerge, SignedBlsToExecutionChange, + SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; #[derive(Debug, Clone, PartialEq)] @@ -239,8 +239,28 @@ impl PubsubMessage { Ok(PubsubMessage::ProposerSlashing(Box::new(proposer_slashing))) } GossipKind::AttesterSlashing => { - let attester_slashing = AttesterSlashing::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?; + // TODO(electra): could an older attester slashing still be floating around during the fork transition? + let attester_slashing = + match fork_context.from_context_bytes(gossip_topic.fork_digest) { + Some(ForkName::Base) + | Some(ForkName::Altair) + | Some(ForkName::Merge) + | Some(ForkName::Capella) + | Some(ForkName::Deneb) => AttesterSlashing::Base( + AttesterSlashingBase::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ), + Some(ForkName::Electra) => AttesterSlashing::Electra( + AttesterSlashingElectra::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ), + None => { + return Err(format!( + "Unknown gossipsub fork digest: {:?}", + gossip_topic.fork_digest + )) + } + }; Ok(PubsubMessage::AttesterSlashing(Box::new(attester_slashing))) } GossipKind::SignedContributionAndProof => { diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 772c2efc944..dab5310f2b5 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1428,7 +1428,7 @@ impl NetworkBeaconProcessor { self.chain .validator_monitor .read() - .register_gossip_attester_slashing(slashing.as_inner()); + .register_gossip_attester_slashing(slashing.as_inner().to_ref()); self.chain.import_attester_slashing(slashing); debug!(self.log, "Successfully imported attester slashing"); diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index 90fdf3cdb81..927e8c9b12c 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -2,8 +2,9 @@ use crate::AttestationStats; use itertools::Itertools; use std::collections::HashMap; use types::{ - attestation::{AttestationBase, AttestationElectra}, superstruct, AggregateSignature, Attestation, AttestationData, - BeaconState, BitList, BitVector, Checkpoint, Epoch, EthSpec, Hash256, Slot, + attestation::{AttestationBase, AttestationElectra}, + superstruct, AggregateSignature, Attestation, AttestationData, BeaconState, BitList, BitVector, + Checkpoint, Epoch, EthSpec, Hash256, Slot, }; #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] @@ -121,12 +122,14 @@ impl<'a, E: EthSpec> AttestationRef<'a, E> { data: self.attestation_data(), signature: indexed_att.signature.clone(), }), - CompactIndexedAttestation::Electra(indexed_att) => Attestation::Electra(AttestationElectra { - aggregation_bits: indexed_att.aggregation_bits.clone(), - data: self.attestation_data(), - signature: indexed_att.signature.clone(), - committee_bits: indexed_att.committee_bits.clone(), - }), + CompactIndexedAttestation::Electra(indexed_att) => { + Attestation::Electra(AttestationElectra { + aggregation_bits: indexed_att.aggregation_bits.clone(), + data: self.attestation_data(), + signature: indexed_att.signature.clone(), + committee_bits: indexed_att.committee_bits.clone(), + }) + } } } } diff --git a/beacon_node/operation_pool/src/attester_slashing.rs b/beacon_node/operation_pool/src/attester_slashing.rs index 725d4d2a857..c2411d4d726 100644 --- a/beacon_node/operation_pool/src/attester_slashing.rs +++ b/beacon_node/operation_pool/src/attester_slashing.rs @@ -1,17 +1,17 @@ use crate::max_cover::MaxCover; use state_processing::per_block_processing::get_slashable_indices_modular; use std::collections::{HashMap, HashSet}; -use types::{AttesterSlashing, BeaconState, EthSpec}; +use types::{AttesterSlashing, AttesterSlashingRef, BeaconState, EthSpec}; #[derive(Debug, Clone)] pub struct AttesterSlashingMaxCover<'a, E: EthSpec> { - slashing: &'a AttesterSlashing, + slashing: AttesterSlashingRef<'a, E>, effective_balances: HashMap, } impl<'a, E: EthSpec> AttesterSlashingMaxCover<'a, E> { pub fn new( - slashing: &'a AttesterSlashing, + slashing: AttesterSlashingRef<'a, E>, proposer_slashing_indices: &HashSet, state: &BeaconState, ) -> Option { @@ -39,16 +39,16 @@ impl<'a, E: EthSpec> AttesterSlashingMaxCover<'a, E> { impl<'a, E: EthSpec> MaxCover for AttesterSlashingMaxCover<'a, E> { /// The result type, of which we would eventually like a collection of maximal quality. type Object = AttesterSlashing; - type Intermediate = AttesterSlashing; + type Intermediate = AttesterSlashingRef<'a, E>; /// The type used to represent sets. type Set = HashMap; - fn intermediate(&self) -> &AttesterSlashing { - self.slashing + fn intermediate(&self) -> &AttesterSlashingRef<'a, E> { + &self.slashing } - fn convert_to_object(slashing: &AttesterSlashing) -> AttesterSlashing { - slashing.clone() + fn convert_to_object(slashing: &AttesterSlashingRef<'a, E>) -> AttesterSlashing { + slashing.clone_as_attester_slashing() } /// Get the set of elements covered. @@ -58,7 +58,7 @@ impl<'a, E: EthSpec> MaxCover for AttesterSlashingMaxCover<'a, E> { /// Update the set of items covered, for the inclusion of some object in the solution. fn update_covering_set( &mut self, - _best_slashing: &AttesterSlashing, + _best_slashing: &AttesterSlashingRef<'a, E>, covered_validator_indices: &HashMap, ) { self.effective_balances diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 4b5fc259507..f700eecf6bd 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -428,7 +428,7 @@ impl OperationPool { let relevant_attester_slashings = reader.iter().flat_map(|slashing| { if slashing.signature_is_still_valid(&state.fork()) { - AttesterSlashingMaxCover::new(slashing.as_inner(), to_be_slashed, state) + AttesterSlashingMaxCover::new(slashing.as_inner().to_ref(), to_be_slashed, state) } else { None } @@ -442,7 +442,7 @@ impl OperationPool { .into_iter() .map(|cover| { to_be_slashed.extend(cover.covering_set().keys()); - cover.intermediate().clone() + AttesterSlashingMaxCover::convert_to_object(cover.intermediate()) }) .collect() } @@ -463,16 +463,19 @@ impl OperationPool { // Check that the attestation's signature is still valid wrt the fork version. let signature_ok = slashing.signature_is_still_valid(&head_state.fork()); // Slashings that don't slash any validators can also be dropped. - let slashing_ok = - get_slashable_indices_modular(head_state, slashing.as_inner(), |_, validator| { + let slashing_ok = get_slashable_indices_modular( + head_state, + slashing.as_inner().to_ref(), + |_, validator| { // Declare that a validator is still slashable if they have not exited prior // to the finalized epoch. // // We cannot check the `slashed` field since the `head` is not finalized and // a fork could un-slash someone. validator.exit_epoch > head_state.finalized_checkpoint().epoch - }) - .map_or(false, |indices| !indices.is_empty()); + }, + ) + .map_or(false, |indices| !indices.is_empty()); signature_ok && slashing_ok }); @@ -907,8 +910,8 @@ mod release_tests { }) .unwrap(); - let att1_indices = get_attesting_indices_from_state(&state, &att1).unwrap(); - let att2_indices = get_attesting_indices_from_state(&state, &att2).unwrap(); + let att1_indices = get_attesting_indices_from_state(&state, att1.to_ref()).unwrap(); + let att2_indices = get_attesting_indices_from_state(&state, att2.to_ref()).unwrap(); let att1_split = SplitAttestation::new(att1.clone(), att1_indices); let att2_split = SplitAttestation::new(att2.clone(), att2_indices); @@ -981,7 +984,8 @@ mod release_tests { for (atts, _) in attestations { for (att, _) in atts { - let attesting_indices = get_attesting_indices_from_state(&state, &att).unwrap(); + let attesting_indices = + get_attesting_indices_from_state(&state, att.to_ref()).unwrap(); op_pool.insert_attestation(att, attesting_indices).unwrap(); } } @@ -1051,7 +1055,7 @@ mod release_tests { for (_, aggregate) in attestations { let att = aggregate.unwrap().message.aggregate; - let attesting_indices = get_attesting_indices_from_state(&state, &att).unwrap(); + let attesting_indices = get_attesting_indices_from_state(&state, att.to_ref()).unwrap(); op_pool .insert_attestation(att.clone(), attesting_indices.clone()) .unwrap(); @@ -1139,7 +1143,8 @@ mod release_tests { .collect::>(); for att in aggs1.into_iter().chain(aggs2.into_iter()) { - let attesting_indices = get_attesting_indices_from_state(&state, &att).unwrap(); + let attesting_indices = + get_attesting_indices_from_state(&state, att.to_ref()).unwrap(); op_pool.insert_attestation(att, attesting_indices).unwrap(); } } @@ -1211,7 +1216,8 @@ mod release_tests { .collect::>(); for att in aggs { - let attesting_indices = get_attesting_indices_from_state(&state, &att).unwrap(); + let attesting_indices = + get_attesting_indices_from_state(&state, att.to_ref()).unwrap(); op_pool.insert_attestation(att, attesting_indices).unwrap(); } }; @@ -1306,7 +1312,8 @@ mod release_tests { .collect::>(); for att in aggs { - let attesting_indices = get_attesting_indices_from_state(&state, &att).unwrap(); + let attesting_indices = + get_attesting_indices_from_state(&state, att.to_ref()).unwrap(); op_pool.insert_attestation(att, attesting_indices).unwrap(); } }; @@ -1349,7 +1356,7 @@ mod release_tests { reward_cache.update(&state).unwrap(); for att in best_attestations { - let attesting_indices = get_attesting_indices_from_state(&state, &att).unwrap(); + let attesting_indices = get_attesting_indices_from_state(&state, att.to_ref()).unwrap(); let split_attestation = SplitAttestation::new(att, attesting_indices); let mut fresh_validators_rewards = AttMaxCover::new( split_attestation.as_ref(), diff --git a/beacon_node/operation_pool/src/persistence.rs b/beacon_node/operation_pool/src/persistence.rs index 8e4e5e2898d..6fd8a6cc3cc 100644 --- a/beacon_node/operation_pool/src/persistence.rs +++ b/beacon_node/operation_pool/src/persistence.rs @@ -39,9 +39,7 @@ pub struct PersistedOperationPool { pub attestations: Vec<(Attestation, Vec)>, /// Mapping from sync contribution ID to sync contributions and aggregate. pub sync_contributions: PersistedSyncContributions, - /// [DEPRECATED] Attester slashings. - #[superstruct(only(V5))] - pub attester_slashings_v5: Vec<(AttesterSlashing, ForkVersion)>, + /// TODO(electra): we've made a DB change here!!! /// Attester slashings. #[superstruct(only(V12, V14, V15))] pub attester_slashings: Vec, E>>, diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index f7836fb2fb4..147fdb01d02 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -14,8 +14,8 @@ use std::marker::PhantomData; use std::time::Duration; use types::{ consts::merge::INTERVALS_PER_SLOT, AbstractExecPayload, AttestationShufflingId, - AttesterSlashing, BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, Checkpoint, Epoch, - EthSpec, ExecPayload, ExecutionBlockHash, Hash256, IndexedAttestation, RelativeEpoch, + AttesterSlashingRef, BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, Checkpoint, + Epoch, EthSpec, ExecPayload, ExecutionBlockHash, Hash256, IndexedAttestationRef, RelativeEpoch, SignedBeaconBlock, Slot, }; @@ -238,8 +238,8 @@ pub struct QueuedAttestation { target_epoch: Epoch, } -impl From<&IndexedAttestation> for QueuedAttestation { - fn from(a: &IndexedAttestation) -> Self { +impl<'a, E: EthSpec> From> for QueuedAttestation { + fn from(a: IndexedAttestationRef<'a, E>) -> Self { Self { slot: a.data().slot, attesting_indices: a.attesting_indices_to_vec(), @@ -940,7 +940,7 @@ where /// https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/fork-choice.md#validate_on_attestation fn validate_on_attestation( &self, - indexed_attestation: &IndexedAttestation, + indexed_attestation: IndexedAttestationRef, is_from_block: AttestationFromBlock, ) -> Result<(), InvalidAttestation> { // There is no point in processing an attestation with an empty bitfield. Reject @@ -1037,7 +1037,7 @@ where pub fn on_attestation( &mut self, system_time_current_slot: Slot, - attestation: &IndexedAttestation, + attestation: IndexedAttestationRef, is_from_block: AttestationFromBlock, ) -> Result<(), Error> { self.update_time(system_time_current_slot)?; @@ -1086,14 +1086,14 @@ where /// Apply an attester slashing to fork choice. /// /// We assume that the attester slashing provided to this function has already been verified. - pub fn on_attester_slashing(&mut self, slashing: &AttesterSlashing) { - let attesting_indices_set = |att: &IndexedAttestation| { + pub fn on_attester_slashing(&mut self, slashing: AttesterSlashingRef<'_, E>) { + let attesting_indices_set = |att: IndexedAttestationRef<'_, E>| { att.attesting_indices_iter() .copied() .collect::>() }; - let att1_indices = attesting_indices_set(&slashing.attestation_1); - let att2_indices = attesting_indices_set(&slashing.attestation_2); + let att1_indices = attesting_indices_set(slashing.attestation_1()); + let att2_indices = attesting_indices_set(slashing.attestation_2()); self.fc_store .extend_equivocating_indices(att1_indices.intersection(&att2_indices).copied()); } diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 6b8ce6f9372..e798f55f844 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -25,14 +25,14 @@ pub fn get_attesting_indices( /// Shortcut for getting the attesting indices while fetching the committee from the state's cache. pub fn get_attesting_indices_from_state( state: &BeaconState, - att: &Attestation, + att: AttestationRef, ) -> Result, BeaconStateError> { let committee = state.get_beacon_committee(att.data().slot, att.data().index)?; match att { - Attestation::Base(att) => { + AttestationRef::Base(att) => { get_attesting_indices::(committee.committee, &att.aggregation_bits) } // TODO(electra) implement get_attesting_indices for electra - Attestation::Electra(_) => todo!(), + AttestationRef::Electra(_) => todo!(), } } diff --git a/consensus/state_processing/src/common/get_indexed_attestation.rs b/consensus/state_processing/src/common/get_indexed_attestation.rs index d6c7512961c..a04d9198c68 100644 --- a/consensus/state_processing/src/common/get_indexed_attestation.rs +++ b/consensus/state_processing/src/common/get_indexed_attestation.rs @@ -9,12 +9,12 @@ type Result = std::result::Result>; /// Spec v0.12.1 pub fn get_indexed_attestation( committee: &[usize], - attestation: &Attestation, + attestation: AttestationRef, ) -> Result> { let attesting_indices = match attestation { - Attestation::Base(att) => get_attesting_indices::(committee, &att.aggregation_bits)?, + AttestationRef::Base(att) => get_attesting_indices::(committee, &att.aggregation_bits)?, // TODO(electra) implement get_attesting_indices for electra - Attestation::Electra(_) => todo!(), + AttestationRef::Electra(_) => todo!(), }; Ok(IndexedAttestation::Base(IndexedAttestationBase { diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index b1196c942d3..b28f218fc8a 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -4,8 +4,9 @@ use crate::EpochCacheError; use std::collections::{hash_map::Entry, HashMap}; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, Attestation, AttestationData, BeaconState, BeaconStateError, BitList, - ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, SignedBeaconBlock, Slot, + AbstractExecPayload, AttestationData, AttestationRef, BeaconState, BeaconStateError, BitList, + ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, IndexedAttestationRef, + SignedBeaconBlock, Slot, }; #[derive(Debug, PartialEq, Clone)] @@ -153,13 +154,13 @@ impl ConsensusContext { } } - pub fn get_indexed_attestation( - &mut self, + pub fn get_indexed_attestation<'a>( + &'a mut self, state: &BeaconState, - attestation: &Attestation, - ) -> Result<&IndexedAttestation, BlockOperationError> { + attestation: AttestationRef<'a, E>, + ) -> Result, BlockOperationError> { let aggregation_bits = match attestation { - Attestation::Base(attn) => { + AttestationRef::Base(attn) => { let mut extended_aggregation_bits: BitList = BitList::with_capacity(attn.aggregation_bits.len()) .map_err(BeaconStateError::from)?; @@ -171,7 +172,7 @@ impl ConsensusContext { } extended_aggregation_bits } - Attestation::Electra(attn) => attn.aggregation_bits.clone(), + AttestationRef::Electra(attn) => attn.aggregation_bits.clone(), }; let key = (attestation.data().clone(), aggregation_bits); @@ -186,6 +187,7 @@ impl ConsensusContext { Ok(vacant.insert(indexed_attestation)) } } + .map(|indexed_attestation| (*indexed_attestation).to_ref()) } pub fn num_cached_indexed_attestations(&self) -> usize { diff --git a/consensus/state_processing/src/lib.rs b/consensus/state_processing/src/lib.rs index 74f9d84bb11..adabf6862d3 100644 --- a/consensus/state_processing/src/lib.rs +++ b/consensus/state_processing/src/lib.rs @@ -45,4 +45,4 @@ pub use per_epoch_processing::{ }; pub use per_slot_processing::{per_slot_processing, Error as SlotProcessingError}; pub use types::{EpochCache, EpochCacheError, EpochCacheKey}; -pub use verify_operation::{SigVerifiedOp, VerifyOperation, VerifyOperationAt}; +pub use verify_operation::{SigVerifiedOp, TransformPersist, VerifyOperation, VerifyOperationAt}; diff --git a/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs b/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs index 884fe8305ba..74477f5e481 100644 --- a/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs +++ b/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs @@ -247,13 +247,12 @@ where ) -> Result<()> { self.sets .sets - .reserve(block.message().body().attester_slashings().len() * 2); + .reserve(block.message().body().attester_slashings_len() * 2); block .message() .body() .attester_slashings() - .iter() .try_for_each(|attester_slashing| { let (set_1, set_2) = attester_slashing_signature_sets( self.state, @@ -277,13 +276,12 @@ where ) -> Result<()> { self.sets .sets - .reserve(block.message().body().attestations().len()); + .reserve(block.message().body().attestations_len()); block .message() .body() .attestations() - .iter() .try_for_each(|attestation| { let indexed_attestation = ctxt.get_indexed_attestation(self.state, attestation)?; diff --git a/consensus/state_processing/src/per_block_processing/is_valid_indexed_attestation.rs b/consensus/state_processing/src/per_block_processing/is_valid_indexed_attestation.rs index 7c7c9e474ac..4bad3315cc4 100644 --- a/consensus/state_processing/src/per_block_processing/is_valid_indexed_attestation.rs +++ b/consensus/state_processing/src/per_block_processing/is_valid_indexed_attestation.rs @@ -13,7 +13,7 @@ fn error(reason: Invalid) -> BlockOperationError { /// Verify an `IndexedAttestation`. pub fn is_valid_indexed_attestation( state: &BeaconState, - indexed_attestation: &IndexedAttestation, + indexed_attestation: IndexedAttestationRef, verify_signatures: VerifySignatures, spec: &ChainSpec, ) -> Result<()> { diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index cb677f5514f..fdccf5af205 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -46,13 +46,16 @@ pub mod base { /// /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. - pub fn process_attestations( + pub fn process_attestations<'a, E: EthSpec, I>( state: &mut BeaconState, - attestations: &[Attestation], + attestations: I, verify_signatures: VerifySignatures, ctxt: &mut ConsensusContext, spec: &ChainSpec, - ) -> Result<(), BlockProcessingError> { + ) -> Result<(), BlockProcessingError> + where + I: Iterator>, + { // Ensure required caches are all built. These should be no-ops during regular operation. state.build_committee_cache(RelativeEpoch::Current, spec)?; state.build_committee_cache(RelativeEpoch::Previous, spec)?; @@ -63,7 +66,7 @@ pub mod base { let proposer_index = ctxt.get_proposer_index(state, spec)?; // Verify and apply each attestation. - for (i, attestation) in attestations.iter().enumerate() { + for (i, attestation) in attestations.enumerate() { verify_attestation_for_block_inclusion( state, attestation, @@ -74,7 +77,7 @@ pub mod base { .map_err(|e| e.into_with_index(i))?; match attestation { - Attestation::Base(att) => { + AttestationRef::Base(att) => { let pending_attestation = PendingAttestation { aggregation_bits: att.aggregation_bits.clone(), data: att.data.clone(), @@ -94,7 +97,7 @@ pub mod base { .push(pending_attestation)?; } } - Attestation::Electra(_) => { + AttestationRef::Electra(_) => { // TODO(electra) pending attestations are only phase 0 // so we should just raise a relevant error here todo!() @@ -110,24 +113,24 @@ pub mod altair_deneb { use super::*; use crate::common::update_progressive_balances_cache::update_progressive_balances_on_attestation; - pub fn process_attestations( + pub fn process_attestations<'a, E: EthSpec, I>( state: &mut BeaconState, - attestations: &[Attestation], + attestations: I, verify_signatures: VerifySignatures, ctxt: &mut ConsensusContext, spec: &ChainSpec, - ) -> Result<(), BlockProcessingError> { - attestations - .iter() - .enumerate() - .try_for_each(|(i, attestation)| { - process_attestation(state, attestation, i, ctxt, verify_signatures, spec) - }) + ) -> Result<(), BlockProcessingError> + where + I: Iterator>, + { + attestations.enumerate().try_for_each(|(i, attestation)| { + process_attestation(state, attestation, i, ctxt, verify_signatures, spec) + }) } pub fn process_attestation( state: &mut BeaconState, - attestation: &Attestation, + attestation: AttestationRef, att_index: usize, ctxt: &mut ConsensusContext, verify_signatures: VerifySignatures, @@ -238,16 +241,19 @@ pub fn process_proposer_slashings( /// /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. -pub fn process_attester_slashings( +pub fn process_attester_slashings<'a, E: EthSpec, I>( state: &mut BeaconState, - attester_slashings: &[AttesterSlashing], + attester_slashings: I, verify_signatures: VerifySignatures, ctxt: &mut ConsensusContext, spec: &ChainSpec, -) -> Result<(), BlockProcessingError> { +) -> Result<(), BlockProcessingError> +where + I: Iterator>, +{ state.build_slashings_cache()?; - for (i, attester_slashing) in attester_slashings.iter().enumerate() { + for (i, attester_slashing) in attester_slashings.enumerate() { let slashable_indices = verify_attester_slashing(state, attester_slashing, verify_signatures, spec) .map_err(|e| e.into_with_index(i))?; diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index 19351a2c2f8..f19714dc193 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -7,10 +7,10 @@ use ssz::DecodeError; use std::borrow::Cow; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, AggregateSignature, AttesterSlashing, BeaconBlockRef, BeaconState, + AbstractExecPayload, AggregateSignature, AttesterSlashingRef, BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, DepositData, Domain, Epoch, EthSpec, Fork, Hash256, - InconsistentFork, IndexedAttestation, ProposerSlashing, PublicKey, PublicKeyBytes, Signature, - SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockHeader, + InconsistentFork, IndexedAttestation, IndexedAttestationRef, ProposerSlashing, PublicKey, + PublicKeyBytes, Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlsToExecutionChange, SignedContributionAndProof, SignedRoot, SignedVoluntaryExit, SigningData, Slot, SyncAggregate, SyncAggregatorSelectionData, Unsigned, }; @@ -272,7 +272,7 @@ pub fn indexed_attestation_signature_set<'a, 'b, E, F>( state: &'a BeaconState, get_pubkey: F, signature: &'a AggregateSignature, - indexed_attestation: &'b IndexedAttestation, + indexed_attestation: IndexedAttestationRef<'b, E>, spec: &'a ChainSpec, ) -> Result> where @@ -335,7 +335,7 @@ where pub fn attester_slashing_signature_sets<'a, E, F>( state: &'a BeaconState, get_pubkey: F, - attester_slashing: &'a AttesterSlashing, + attester_slashing: AttesterSlashingRef<'a, E>, spec: &'a ChainSpec, ) -> Result<(SignatureSet<'a>, SignatureSet<'a>)> where @@ -346,15 +346,15 @@ where indexed_attestation_signature_set( state, get_pubkey.clone(), - attester_slashing.attestation_1.signature(), - &attester_slashing.attestation_1, + attester_slashing.attestation_1().signature(), + attester_slashing.attestation_1(), spec, )?, indexed_attestation_signature_set( state, get_pubkey, - attester_slashing.attestation_2.signature(), - &attester_slashing.attestation_2, + attester_slashing.attestation_2().signature(), + attester_slashing.attestation_2(), spec, )?, )) diff --git a/consensus/state_processing/src/per_block_processing/tests.rs b/consensus/state_processing/src/per_block_processing/tests.rs index 85bb740a6cd..2774dd3d87f 100644 --- a/consensus/state_processing/src/per_block_processing/tests.rs +++ b/consensus/state_processing/src/per_block_processing/tests.rs @@ -388,7 +388,12 @@ async fn invalid_attestation_no_committee_for_index() { .clone() .deconstruct() .0; - head_block.to_mut().body_mut().attestations_mut()[0] + head_block + .to_mut() + .body_mut() + .attestations_mut() + .next() + .unwrap() .data_mut() .index += 1; let mut ctxt = ConsensusContext::new(state.slot()); @@ -423,10 +428,21 @@ async fn invalid_attestation_wrong_justified_checkpoint() { .clone() .deconstruct() .0; - let old_justified_checkpoint = head_block.body().attestations()[0].data().source; + let old_justified_checkpoint = head_block + .body() + .attestations() + .next() + .unwrap() + .data() + .source; let mut new_justified_checkpoint = old_justified_checkpoint; new_justified_checkpoint.epoch += Epoch::new(1); - head_block.to_mut().body_mut().attestations_mut()[0] + head_block + .to_mut() + .body_mut() + .attestations_mut() + .next() + .unwrap() .data_mut() .source = new_justified_checkpoint; @@ -467,7 +483,12 @@ async fn invalid_attestation_bad_aggregation_bitfield_len() { .clone() .deconstruct() .0; - *head_block.to_mut().body_mut().attestations_mut()[0] + *head_block + .to_mut() + .body_mut() + .attestations_mut() + .next() + .unwrap() .aggregation_bits_base_mut() .unwrap() = Bitfield::with_capacity(spec.target_committee_size).unwrap(); @@ -502,8 +523,13 @@ async fn invalid_attestation_bad_signature() { .clone() .deconstruct() .0; - *head_block.to_mut().body_mut().attestations_mut()[0].signature_mut() = - AggregateSignature::empty(); + *head_block + .to_mut() + .body_mut() + .attestations_mut() + .next() + .unwrap() + .signature_mut() = AggregateSignature::empty(); let mut ctxt = ConsensusContext::new(state.slot()); let result = process_operations::process_attestations( @@ -538,9 +564,14 @@ async fn invalid_attestation_included_too_early() { .clone() .deconstruct() .0; - let new_attesation_slot = head_block.body().attestations()[0].data().slot + let new_attesation_slot = head_block.body().attestations().next().unwrap().data().slot + Slot::new(MainnetEthSpec::slots_per_epoch()); - head_block.to_mut().body_mut().attestations_mut()[0] + head_block + .to_mut() + .body_mut() + .attestations_mut() + .next() + .unwrap() .data_mut() .slot = new_attesation_slot; @@ -581,9 +612,14 @@ async fn invalid_attestation_included_too_late() { .clone() .deconstruct() .0; - let new_attesation_slot = head_block.body().attestations()[0].data().slot + let new_attesation_slot = head_block.body().attestations().next().unwrap().data().slot - Slot::new(MainnetEthSpec::slots_per_epoch()); - head_block.to_mut().body_mut().attestations_mut()[0] + head_block + .to_mut() + .body_mut() + .attestations_mut() + .next() + .unwrap() .data_mut() .slot = new_attesation_slot; @@ -621,7 +657,12 @@ async fn invalid_attestation_target_epoch_slot_mismatch() { .clone() .deconstruct() .0; - head_block.to_mut().body_mut().attestations_mut()[0] + head_block + .to_mut() + .body_mut() + .attestations_mut() + .next() + .unwrap() .data_mut() .target .epoch += Epoch::new(1); @@ -657,7 +698,7 @@ async fn valid_insert_attester_slashing() { let mut ctxt = ConsensusContext::new(state.slot()); let result = process_operations::process_attester_slashings( &mut state, - &[attester_slashing], + [attester_slashing.to_ref()].into_iter(), VerifySignatures::True, &mut ctxt, &spec, @@ -673,13 +714,20 @@ async fn invalid_attester_slashing_not_slashable() { let harness = get_harness::(EPOCH_OFFSET, VALIDATOR_COUNT).await; let mut attester_slashing = harness.make_attester_slashing(vec![1, 2]); - attester_slashing.attestation_1 = attester_slashing.attestation_2.clone(); + match &mut attester_slashing { + AttesterSlashing::Base(ref mut attester_slashing) => { + attester_slashing.attestation_1 = attester_slashing.attestation_2.clone(); + } + AttesterSlashing::Electra(ref mut attester_slashing) => { + attester_slashing.attestation_1 = attester_slashing.attestation_2.clone(); + } + } let mut state = harness.get_current_state(); let mut ctxt = ConsensusContext::new(state.slot()); let result = process_operations::process_attester_slashings( &mut state, - &[attester_slashing], + [attester_slashing.to_ref()].into_iter(), VerifySignatures::True, &mut ctxt, &spec, @@ -701,16 +749,20 @@ async fn invalid_attester_slashing_1_invalid() { let harness = get_harness::(EPOCH_OFFSET, VALIDATOR_COUNT).await; let mut attester_slashing = harness.make_attester_slashing(vec![1, 2]); - *attester_slashing - .attestation_1 - .attesting_indices_base_mut() - .unwrap() = VariableList::from(vec![2, 1]); + match &mut attester_slashing { + AttesterSlashing::Base(ref mut attester_slashing) => { + attester_slashing.attestation_1.attesting_indices = VariableList::from(vec![2, 1]); + } + AttesterSlashing::Electra(ref mut attester_slashing) => { + attester_slashing.attestation_1.attesting_indices = VariableList::from(vec![2, 1]); + } + } let mut state = harness.get_current_state(); let mut ctxt = ConsensusContext::new(state.slot()); let result = process_operations::process_attester_slashings( &mut state, - &[attester_slashing], + [attester_slashing.to_ref()].into_iter(), VerifySignatures::True, &mut ctxt, &spec, @@ -735,16 +787,20 @@ async fn invalid_attester_slashing_2_invalid() { let harness = get_harness::(EPOCH_OFFSET, VALIDATOR_COUNT).await; let mut attester_slashing = harness.make_attester_slashing(vec![1, 2]); - *attester_slashing - .attestation_2 - .attesting_indices_base_mut() - .unwrap() = VariableList::from(vec![2, 1]); + match &mut attester_slashing { + AttesterSlashing::Base(ref mut attester_slashing) => { + attester_slashing.attestation_2.attesting_indices = VariableList::from(vec![2, 1]); + } + AttesterSlashing::Electra(ref mut attester_slashing) => { + attester_slashing.attestation_2.attesting_indices = VariableList::from(vec![2, 1]); + } + } let mut state = harness.get_current_state(); let mut ctxt = ConsensusContext::new(state.slot()); let result = process_operations::process_attester_slashings( &mut state, - &[attester_slashing], + [attester_slashing.to_ref()].into_iter(), VerifySignatures::True, &mut ctxt, &spec, diff --git a/consensus/state_processing/src/per_block_processing/verify_attestation.rs b/consensus/state_processing/src/per_block_processing/verify_attestation.rs index 8369f988f73..df9bbc855c1 100644 --- a/consensus/state_processing/src/per_block_processing/verify_attestation.rs +++ b/consensus/state_processing/src/per_block_processing/verify_attestation.rs @@ -17,11 +17,11 @@ fn error(reason: Invalid) -> BlockOperationError { /// Optionally verifies the aggregate signature, depending on `verify_signatures`. pub fn verify_attestation_for_block_inclusion<'ctxt, E: EthSpec>( state: &BeaconState, - attestation: &Attestation, + attestation: AttestationRef<'ctxt, E>, ctxt: &'ctxt mut ConsensusContext, verify_signatures: VerifySignatures, spec: &ChainSpec, -) -> Result<&'ctxt IndexedAttestation> { +) -> Result> { let data = attestation.data(); verify!( @@ -61,11 +61,11 @@ pub fn verify_attestation_for_block_inclusion<'ctxt, E: EthSpec>( /// Spec v0.12.1 pub fn verify_attestation_for_state<'ctxt, E: EthSpec>( state: &BeaconState, - attestation: &Attestation, + attestation: AttestationRef<'ctxt, E>, ctxt: &'ctxt mut ConsensusContext, verify_signatures: VerifySignatures, spec: &ChainSpec, -) -> Result<&'ctxt IndexedAttestation> { +) -> Result> { let data = attestation.data(); verify!( @@ -87,7 +87,7 @@ pub fn verify_attestation_for_state<'ctxt, E: EthSpec>( /// /// Spec v0.12.1 fn verify_casper_ffg_vote( - attestation: &Attestation, + attestation: AttestationRef, state: &BeaconState, ) -> Result<()> { let data = attestation.data(); diff --git a/consensus/state_processing/src/per_block_processing/verify_attester_slashing.rs b/consensus/state_processing/src/per_block_processing/verify_attester_slashing.rs index 7fb784a968e..7fe4c8bc08b 100644 --- a/consensus/state_processing/src/per_block_processing/verify_attester_slashing.rs +++ b/consensus/state_processing/src/per_block_processing/verify_attester_slashing.rs @@ -18,12 +18,12 @@ fn error(reason: Invalid) -> BlockOperationError { /// invalidity. pub fn verify_attester_slashing( state: &BeaconState, - attester_slashing: &AttesterSlashing, + attester_slashing: AttesterSlashingRef<'_, E>, verify_signatures: VerifySignatures, spec: &ChainSpec, ) -> Result> { - let attestation_1 = &attester_slashing.attestation_1; - let attestation_2 = &attester_slashing.attestation_2; + let attestation_1 = attester_slashing.attestation_1(); + let attestation_2 = attester_slashing.attestation_2(); // Spec: is_slashable_attestation_data verify!( @@ -45,7 +45,7 @@ pub fn verify_attester_slashing( /// Returns Ok(indices) if `indices.len() > 0` pub fn get_slashable_indices( state: &BeaconState, - attester_slashing: &AttesterSlashing, + attester_slashing: AttesterSlashingRef<'_, E>, ) -> Result> { get_slashable_indices_modular(state, attester_slashing, |_, validator| { validator.is_slashable_at(state.current_epoch()) @@ -56,14 +56,14 @@ pub fn get_slashable_indices( /// for determining whether a given validator should be considered slashable. pub fn get_slashable_indices_modular( state: &BeaconState, - attester_slashing: &AttesterSlashing, + attester_slashing: AttesterSlashingRef<'_, E>, is_slashable: F, ) -> Result> where F: Fn(u64, &Validator) -> bool, { - let attestation_1 = &attester_slashing.attestation_1; - let attestation_2 = &attester_slashing.attestation_2; + let attestation_1 = attester_slashing.attestation_1(); + let attestation_2 = attester_slashing.attestation_2(); let attesting_indices_1 = attestation_1 .attesting_indices_iter() diff --git a/consensus/state_processing/src/verify_operation.rs b/consensus/state_processing/src/verify_operation.rs index 4a2e5e22c56..c12467fbb57 100644 --- a/consensus/state_processing/src/verify_operation.rs +++ b/consensus/state_processing/src/verify_operation.rs @@ -12,30 +12,132 @@ use smallvec::{smallvec, SmallVec}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::marker::PhantomData; +use types::{AttesterSlashing, AttesterSlashingOnDisk, AttesterSlashingRefOnDisk}; use types::{ - AttesterSlashing, BeaconState, ChainSpec, Epoch, EthSpec, Fork, ForkVersion, ProposerSlashing, + BeaconState, ChainSpec, Epoch, EthSpec, Fork, ForkVersion, ProposerSlashing, SignedBlsToExecutionChange, SignedVoluntaryExit, }; const MAX_FORKS_VERIFIED_AGAINST: usize = 2; +pub trait TransformPersist { + type Persistable: Encode + Decode; + type PersistableRef<'a>: Encode + where + Self: 'a; + + /// Returns a reference to the object in a form that implements `Encode` + fn as_persistable_ref(&self) -> Self::PersistableRef<'_>; + + /// Converts the object back into its original form. + fn from_persistable(persistable: Self::Persistable) -> Self; +} + /// Wrapper around an operation type that acts as proof that its signature has been checked. /// /// The inner `op` field is private, meaning instances of this type can only be constructed /// by calling `validate`. -#[derive(Derivative, Debug, Clone, Encode, Decode)] +#[derive(Derivative, Debug, Clone)] #[derivative( PartialEq, Eq, - Hash(bound = "T: Encode + Decode + std::hash::Hash, E: EthSpec") + Hash(bound = "T: TransformPersist + std::hash::Hash, E: EthSpec") )] -pub struct SigVerifiedOp { +pub struct SigVerifiedOp { op: T, verified_against: VerifiedAgainst, - #[ssz(skip_serializing, skip_deserializing)] + //#[ssz(skip_serializing, skip_deserializing)] _phantom: PhantomData, } +impl Encode for SigVerifiedOp { + fn is_ssz_fixed_len() -> bool { + ::is_ssz_fixed_len() + && ::is_ssz_fixed_len() + } + + #[allow(clippy::expect_used)] + fn ssz_fixed_len() -> usize { + if ::is_ssz_fixed_len() { + ::ssz_fixed_len() + .checked_add(::ssz_fixed_len()) + .expect("encode ssz_fixed_len length overflow") + } else { + ssz::BYTES_PER_LENGTH_OFFSET + } + } + + #[allow(clippy::expect_used)] + fn ssz_bytes_len(&self) -> usize { + if ::is_ssz_fixed_len() { + ::ssz_fixed_len() + } else { + let persistable = self.op.as_persistable_ref(); + persistable + .ssz_bytes_len() + .checked_add(self.verified_against.ssz_bytes_len()) + .expect("ssz_bytes_len length overflow") + } + } + + fn ssz_append(&self, buf: &mut Vec) { + let mut encoder = ssz::SszEncoder::container(buf, ::ssz_fixed_len()); + let persistable = self.op.as_persistable_ref(); + encoder.append(&persistable); + encoder.append(&self.verified_against); + encoder.finalize(); + } +} + +impl Decode for SigVerifiedOp { + fn is_ssz_fixed_len() -> bool { + ::is_ssz_fixed_len() + && ::is_ssz_fixed_len() + } + + #[allow(clippy::expect_used)] + fn ssz_fixed_len() -> usize { + if ::is_ssz_fixed_len() { + ::ssz_fixed_len() + .checked_add(::ssz_fixed_len()) + .expect("decode ssz_fixed_len length overflow") + } else { + ssz::BYTES_PER_LENGTH_OFFSET + } + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let mut builder = ssz::SszDecoderBuilder::new(bytes); + + // Register types based on whether they are fixed or variable length + if ::is_ssz_fixed_len() { + builder.register_type::()?; + } else { + builder.register_anonymous_variable_length_item()?; + } + + if ::is_ssz_fixed_len() { + builder.register_type::()?; + } else { + builder.register_anonymous_variable_length_item()?; + } + + let mut decoder = builder.build()?; + // Decode each component + let persistable: T::Persistable = decoder.decode_next()?; + let verified_against: VerifiedAgainst = decoder.decode_next()?; + + // Use TransformPersist to convert persistable back into the original type + let op = T::from_persistable(persistable); + + Ok(SigVerifiedOp { + op, + verified_against, + _phantom: PhantomData, + }) + } +} + /// Information about the fork versions that this message was verified against. /// /// In general it is not safe to assume that a `SigVerifiedOp` constructed at some point in the past @@ -109,7 +211,7 @@ where } /// Trait for operations that can be verified and transformed into a `SigVerifiedOp`. -pub trait VerifyOperation: Encode + Decode + Sized { +pub trait VerifyOperation: TransformPersist + Sized { type Error; fn validate( @@ -152,15 +254,15 @@ impl VerifyOperation for AttesterSlashing { state: &BeaconState, spec: &ChainSpec, ) -> Result, Self::Error> { - verify_attester_slashing(state, &self, VerifySignatures::True, spec)?; + verify_attester_slashing(state, self.to_ref(), VerifySignatures::True, spec)?; Ok(SigVerifiedOp::new(self, state)) } #[allow(clippy::arithmetic_side_effects)] fn verification_epochs(&self) -> SmallVec<[Epoch; MAX_FORKS_VERIFIED_AGAINST]> { smallvec![ - self.attestation_1.data().target.epoch, - self.attestation_2.data().target.epoch + self.attestation_1().data().target.epoch, + self.attestation_2().data().target.epoch ] } } @@ -237,3 +339,55 @@ impl VerifyOperationAt for SignedVoluntaryExit { Ok(SigVerifiedOp::new(self, state)) } } + +impl TransformPersist for SignedVoluntaryExit { + type Persistable = Self; + type PersistableRef<'a> = &'a Self; + + fn as_persistable_ref(&self) -> Self::PersistableRef<'_> { + self + } + + fn from_persistable(persistable: Self::Persistable) -> Self { + persistable + } +} + +impl TransformPersist for AttesterSlashing { + type Persistable = AttesterSlashingOnDisk; + type PersistableRef<'a> = AttesterSlashingRefOnDisk<'a, E>; + + fn as_persistable_ref(&self) -> Self::PersistableRef<'_> { + self.to_ref().into() + } + + fn from_persistable(persistable: Self::Persistable) -> Self { + persistable.into() + } +} + +impl TransformPersist for ProposerSlashing { + type Persistable = Self; + type PersistableRef<'a> = &'a Self; + + fn as_persistable_ref(&self) -> Self::PersistableRef<'_> { + self + } + + fn from_persistable(persistable: Self::Persistable) -> Self { + persistable + } +} + +impl TransformPersist for SignedBlsToExecutionChange { + type Persistable = Self; + type PersistableRef<'a> = &'a Self; + + fn as_persistable_ref(&self) -> Self::PersistableRef<'_> { + self + } + + fn from_persistable(persistable: Self::Persistable) -> Self { + persistable + } +} diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index e036f958fff..0d3f29ae289 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -212,6 +212,15 @@ impl Attestation { } } +impl<'a, E: EthSpec> AttestationRef<'a, E> { + pub fn clone_as_attestation(self) -> Attestation { + match self { + AttestationRef::Base(att) => Attestation::Base(att.clone()), + AttestationRef::Electra(att) => Attestation::Electra(att.clone()), + } + } +} + impl AttestationElectra { /// Are the aggregation bitfields of these attestations disjoint? pub fn signers_disjoint_from(&self, other: &Self) -> bool { @@ -363,6 +372,12 @@ impl SlotData for Attestation { } } +impl<'a, E: EthSpec> SlotData for AttestationRef<'a, E> { + fn get_slot(&self) -> Slot { + self.data().slot + } +} + #[cfg(test)] mod tests { use super::*; @@ -394,5 +409,6 @@ mod tests { ); } - ssz_and_tree_hash_tests!(Attestation); + // TODO(electra): can we do this with both variants or should we? + ssz_and_tree_hash_tests!(AttestationBase); } diff --git a/consensus/types/src/attester_slashing.rs b/consensus/types/src/attester_slashing.rs index 5ad5297d0ce..4ad19f477ca 100644 --- a/consensus/types/src/attester_slashing.rs +++ b/consensus/types/src/attester_slashing.rs @@ -1,32 +1,168 @@ -use crate::{test_utils::TestRandom, EthSpec, IndexedAttestation}; - +use crate::indexed_attestation::{ + IndexedAttestationBase, IndexedAttestationElectra, IndexedAttestationRef, +}; +use crate::{test_utils::TestRandom, EthSpec}; use derivative::Derivative; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; +use superstruct::superstruct; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -/// Two conflicting attestations. -/// -/// Spec v0.12.1 +#[superstruct( + variants(Base, Electra), + variant_attributes( + derive( + Derivative, + Debug, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, + arbitrary::Arbitrary + ), + derivative(PartialEq, Eq, Hash(bound = "E: EthSpec")), + serde(bound = "E: EthSpec"), + arbitrary(bound = "E: EthSpec") + ), + ref_attributes(derive(Debug)) +)] #[derive( - Derivative, - Debug, - Clone, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, - arbitrary::Arbitrary, + Debug, Clone, Serialize, Encode, Deserialize, TreeHash, Derivative, arbitrary::Arbitrary, )] #[derivative(PartialEq, Eq, Hash(bound = "E: EthSpec"))] -#[serde(bound = "E: EthSpec")] +#[serde(bound = "E: EthSpec", untagged)] #[arbitrary(bound = "E: EthSpec")] +#[ssz(enum_behaviour = "transparent")] +#[tree_hash(enum_behaviour = "transparent")] pub struct AttesterSlashing { - pub attestation_1: IndexedAttestation, - pub attestation_2: IndexedAttestation, + // TODO(electra) change this to `#[superstruct(flatten)]` when 0.8 is out.. + #[superstruct(only(Base), partial_getter(rename = "attestation_1_base"))] + pub attestation_1: IndexedAttestationBase, + #[superstruct(only(Electra), partial_getter(rename = "attestation_1_electra"))] + pub attestation_1: IndexedAttestationElectra, + #[superstruct(only(Base), partial_getter(rename = "attestation_2_base"))] + pub attestation_2: IndexedAttestationBase, + #[superstruct(only(Electra), partial_getter(rename = "attestation_2_electra"))] + pub attestation_2: IndexedAttestationElectra, +} + +/// This is a copy of the `AttesterSlashing` enum but with `Encode` and `Decode` derived +/// using the `union` behavior for the purposes of persistence on disk. We use a separate +/// type so that we don't accidentally use this non-spec encoding in consensus objects. +#[derive(Debug, Clone, Encode, Decode, Derivative)] +#[derivative(PartialEq, Eq, Hash(bound = "E: EthSpec"))] +#[ssz(enum_behaviour = "union")] +pub enum AttesterSlashingOnDisk { + Base(AttesterSlashingBase), + Electra(AttesterSlashingElectra), +} + +#[derive(Debug, Clone, Encode)] +#[ssz(enum_behaviour = "union")] +pub enum AttesterSlashingRefOnDisk<'a, E: EthSpec> { + Base(&'a AttesterSlashingBase), + Electra(&'a AttesterSlashingElectra), +} + +impl From> for AttesterSlashingOnDisk { + fn from(attester_slashing: AttesterSlashing) -> Self { + match attester_slashing { + AttesterSlashing::Base(attester_slashing) => Self::Base(attester_slashing), + AttesterSlashing::Electra(attester_slashing) => Self::Electra(attester_slashing), + } + } +} + +impl From> for AttesterSlashing { + fn from(attester_slashing: AttesterSlashingOnDisk) -> Self { + match attester_slashing { + AttesterSlashingOnDisk::Base(attester_slashing) => Self::Base(attester_slashing), + AttesterSlashingOnDisk::Electra(attester_slashing) => Self::Electra(attester_slashing), + } + } +} + +impl<'a, E: EthSpec> From> for AttesterSlashingRef<'a, E> { + fn from(attester_slashing: AttesterSlashingRefOnDisk<'a, E>) -> Self { + match attester_slashing { + AttesterSlashingRefOnDisk::Base(attester_slashing) => Self::Base(attester_slashing), + AttesterSlashingRefOnDisk::Electra(attester_slashing) => { + Self::Electra(attester_slashing) + } + } + } +} + +impl<'a, E: EthSpec> From> for AttesterSlashingRefOnDisk<'a, E> { + fn from(attester_slashing: AttesterSlashingRef<'a, E>) -> Self { + match attester_slashing { + AttesterSlashingRef::Base(attester_slashing) => Self::Base(attester_slashing), + AttesterSlashingRef::Electra(attester_slashing) => Self::Electra(attester_slashing), + } + } +} + +impl<'a, E: EthSpec> AttesterSlashingRef<'a, E> { + pub fn clone_as_attester_slashing(self) -> AttesterSlashing { + match self { + AttesterSlashingRef::Base(attester_slashing) => { + AttesterSlashing::Base(attester_slashing.clone()) + } + AttesterSlashingRef::Electra(attester_slashing) => { + AttesterSlashing::Electra(attester_slashing.clone()) + } + } + } + + pub fn attestation_1(&self) -> IndexedAttestationRef<'a, E> { + match self { + AttesterSlashingRef::Base(attester_slashing) => { + IndexedAttestationRef::Base(&attester_slashing.attestation_1) + } + AttesterSlashingRef::Electra(attester_slashing) => { + IndexedAttestationRef::Electra(&attester_slashing.attestation_1) + } + } + } + + pub fn attestation_2(&self) -> IndexedAttestationRef<'a, E> { + match self { + AttesterSlashingRef::Base(attester_slashing) => { + IndexedAttestationRef::Base(&attester_slashing.attestation_2) + } + AttesterSlashingRef::Electra(attester_slashing) => { + IndexedAttestationRef::Electra(&attester_slashing.attestation_2) + } + } + } +} + +impl AttesterSlashing { + pub fn attestation_1(&self) -> IndexedAttestationRef { + match self { + AttesterSlashing::Base(attester_slashing) => { + IndexedAttestationRef::Base(&attester_slashing.attestation_1) + } + AttesterSlashing::Electra(attester_slashing) => { + IndexedAttestationRef::Electra(&attester_slashing.attestation_1) + } + } + } + + pub fn attestation_2(&self) -> IndexedAttestationRef { + match self { + AttesterSlashing::Base(attester_slashing) => { + IndexedAttestationRef::Base(&attester_slashing.attestation_2) + } + AttesterSlashing::Electra(attester_slashing) => { + IndexedAttestationRef::Electra(&attester_slashing.attestation_2) + } + } + } } #[cfg(test)] @@ -34,5 +170,6 @@ mod tests { use super::*; use crate::*; - ssz_and_tree_hash_tests!(AttesterSlashing); + // TODO(electra): should this be done for both variants? + ssz_and_tree_hash_tests!(AttesterSlashingBase); } diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index dd1a12abba1..045cfb0ef5c 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -11,7 +11,7 @@ use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; -use self::indexed_attestation::IndexedAttestationBase; +use self::indexed_attestation::{IndexedAttestationBase, IndexedAttestationElectra}; /// A block of the `BeaconChain`. #[superstruct( @@ -327,16 +327,15 @@ impl> BeaconBlockBase { message: header, signature: Signature::empty(), }; - let indexed_attestation: IndexedAttestation = - IndexedAttestation::Base(IndexedAttestationBase { - attesting_indices: VariableList::new(vec![ - 0_u64; - E::MaxValidatorsPerCommittee::to_usize() - ]) - .unwrap(), - data: AttestationData::default(), - signature: AggregateSignature::empty(), - }); + let indexed_attestation = IndexedAttestationBase { + attesting_indices: VariableList::new(vec![ + 0_u64; + E::MaxValidatorsPerCommittee::to_usize() + ]) + .unwrap(), + data: AttestationData::default(), + signature: AggregateSignature::empty(), + }; let deposit_data = DepositData { pubkey: PublicKeyBytes::empty(), @@ -349,17 +348,17 @@ impl> BeaconBlockBase { signed_header_2: signed_header, }; - let attester_slashing = AttesterSlashing { + let attester_slashing = AttesterSlashingBase { attestation_1: indexed_attestation.clone(), attestation_2: indexed_attestation, }; - let attestation: Attestation = Attestation::Base(AttestationBase { + let attestation = AttestationBase { aggregation_bits: BitList::with_capacity(E::MaxValidatorsPerCommittee::to_usize()) .unwrap(), data: AttestationData::default(), signature: AggregateSignature::empty(), - }); + }; let deposit = Deposit { proof: FixedVector::from_elem(Hash256::zero()), @@ -607,6 +606,42 @@ impl> BeaconBlockElectra /// Return a Electra block where the block has maximum size. pub fn full(spec: &ChainSpec) -> Self { let base_block: BeaconBlockBase<_, Payload> = BeaconBlockBase::full(spec); + // TODO(electra): check this + let indexed_attestation: IndexedAttestationElectra = IndexedAttestationElectra { + attesting_indices: VariableList::new(vec![ + 0_u64; + E::MaxValidatorsPerCommitteePerSlot::to_usize( + ) + ]) + .unwrap(), + data: AttestationData::default(), + signature: AggregateSignature::empty(), + }; + // TODO(electra): fix this so we calculate this size correctly + let attester_slashings = vec![ + AttesterSlashingElectra { + attestation_1: indexed_attestation.clone(), + attestation_2: indexed_attestation, + }; + E::max_attester_slashings_electra() + ] + .into(); + // TODO(electra): check this + let attestation = AttestationElectra { + aggregation_bits: BitList::with_capacity( + E::MaxValidatorsPerCommitteePerSlot::to_usize(), + ) + .unwrap(), + data: AttestationData::default(), + signature: AggregateSignature::empty(), + // TODO(electra): does this actually allocate the size correctly? + committee_bits: BitVector::new(), + }; + let mut attestations_electra = vec![]; + for _ in 0..E::MaxAttestationsElectra::to_usize() { + attestations_electra.push(attestation.clone()); + } + let bls_to_execution_changes = vec![ SignedBlsToExecutionChange { message: BlsToExecutionChange { @@ -630,8 +665,8 @@ impl> BeaconBlockElectra state_root: Hash256::zero(), body: BeaconBlockBodyElectra { proposer_slashings: base_block.body.proposer_slashings, - attester_slashings: base_block.body.attester_slashings, - attestations: base_block.body.attestations, + attester_slashings, + attestations: attestations_electra.into(), deposits: base_block.body.deposits, voluntary_exits: base_block.body.voluntary_exits, bls_to_execution_changes, @@ -645,6 +680,7 @@ impl> BeaconBlockElectra graffiti: Graffiti::default(), execution_payload: Payload::Electra::default(), blob_kzg_commitments: VariableList::empty(), + consolidations: VariableList::empty(), }, } } @@ -675,6 +711,7 @@ impl> EmptyBlock for BeaconBlockElec execution_payload: Payload::Electra::default(), bls_to_execution_changes: VariableList::empty(), blob_kzg_commitments: VariableList::empty(), + consolidations: VariableList::empty(), }, } } diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index a55c16b80d5..9fdffd0736a 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -63,8 +63,21 @@ pub struct BeaconBlockBody = FullPay pub eth1_data: Eth1Data, pub graffiti: Graffiti, pub proposer_slashings: VariableList, - pub attester_slashings: VariableList, E::MaxAttesterSlashings>, - pub attestations: VariableList, E::MaxAttestations>, + #[superstruct( + only(Base, Altair, Merge, Capella, Deneb), + partial_getter(rename = "attester_slashings_base") + )] + pub attester_slashings: VariableList, E::MaxAttesterSlashings>, + #[superstruct(only(Electra), partial_getter(rename = "attester_slashings_electra"))] + pub attester_slashings: + VariableList, E::MaxAttesterSlashingsElectra>, + #[superstruct( + only(Base, Altair, Merge, Capella, Deneb), + partial_getter(rename = "attestations_base") + )] + pub attestations: VariableList, E::MaxAttestations>, + #[superstruct(only(Electra), partial_getter(rename = "attestations_electra"))] + pub attestations: VariableList, E::MaxAttestationsElectra>, pub deposits: VariableList, pub voluntary_exits: VariableList, #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] @@ -89,6 +102,8 @@ pub struct BeaconBlockBody = FullPay VariableList, #[superstruct(only(Deneb, Electra))] pub blob_kzg_commitments: KzgCommitments, + #[superstruct(only(Electra))] + pub consolidations: VariableList, #[superstruct(only(Base, Altair))] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] @@ -253,6 +268,99 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, self.blob_kzg_commitments() .map_or(false, |blobs| !blobs.is_empty()) } + + pub fn attestations_len(&self) -> usize { + match self { + Self::Base(body) => body.attestations.len(), + Self::Altair(body) => body.attestations.len(), + Self::Merge(body) => body.attestations.len(), + Self::Capella(body) => body.attestations.len(), + Self::Deneb(body) => body.attestations.len(), + Self::Electra(body) => body.attestations.len(), + } + } + + pub fn attester_slashings_len(&self) -> usize { + match self { + Self::Base(body) => body.attester_slashings.len(), + Self::Altair(body) => body.attester_slashings.len(), + Self::Merge(body) => body.attester_slashings.len(), + Self::Capella(body) => body.attester_slashings.len(), + Self::Deneb(body) => body.attester_slashings.len(), + Self::Electra(body) => body.attester_slashings.len(), + } + } + + pub fn attestations(&self) -> Box> + 'a> { + match self { + Self::Base(body) => Box::new(body.attestations.iter().map(AttestationRef::Base)), + Self::Altair(body) => Box::new(body.attestations.iter().map(AttestationRef::Base)), + Self::Merge(body) => Box::new(body.attestations.iter().map(AttestationRef::Base)), + Self::Capella(body) => Box::new(body.attestations.iter().map(AttestationRef::Base)), + Self::Deneb(body) => Box::new(body.attestations.iter().map(AttestationRef::Base)), + Self::Electra(body) => Box::new(body.attestations.iter().map(AttestationRef::Electra)), + } + } + + pub fn attester_slashings(&self) -> Box> + 'a> { + match self { + Self::Base(body) => Box::new( + body.attester_slashings + .iter() + .map(AttesterSlashingRef::Base), + ), + Self::Altair(body) => Box::new( + body.attester_slashings + .iter() + .map(AttesterSlashingRef::Base), + ), + Self::Merge(body) => Box::new( + body.attester_slashings + .iter() + .map(AttesterSlashingRef::Base), + ), + Self::Capella(body) => Box::new( + body.attester_slashings + .iter() + .map(AttesterSlashingRef::Base), + ), + Self::Deneb(body) => Box::new( + body.attester_slashings + .iter() + .map(AttesterSlashingRef::Base), + ), + Self::Electra(body) => Box::new( + body.attester_slashings + .iter() + .map(AttesterSlashingRef::Electra), + ), + } + } +} + +impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRefMut<'a, E, Payload> { + pub fn attestations_mut( + &'a mut self, + ) -> Box> + 'a> { + match self { + Self::Base(body) => Box::new(body.attestations.iter_mut().map(AttestationRefMut::Base)), + Self::Altair(body) => { + Box::new(body.attestations.iter_mut().map(AttestationRefMut::Base)) + } + Self::Merge(body) => { + Box::new(body.attestations.iter_mut().map(AttestationRefMut::Base)) + } + Self::Capella(body) => { + Box::new(body.attestations.iter_mut().map(AttestationRefMut::Base)) + } + Self::Deneb(body) => { + Box::new(body.attestations.iter_mut().map(AttestationRefMut::Base)) + } + Self::Electra(body) => { + Box::new(body.attestations.iter_mut().map(AttestationRefMut::Electra)) + } + } + } } impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, Payload> { @@ -553,6 +661,7 @@ impl From>> execution_payload: FullPayloadElectra { execution_payload }, bls_to_execution_changes, blob_kzg_commitments, + consolidations, } = body; ( @@ -571,6 +680,7 @@ impl From>> }, bls_to_execution_changes, blob_kzg_commitments: blob_kzg_commitments.clone(), + consolidations, }, Some(execution_payload), ) @@ -709,6 +819,7 @@ impl BeaconBlockBodyElectra> { execution_payload: FullPayloadElectra { execution_payload }, bls_to_execution_changes, blob_kzg_commitments, + consolidations, } = self; BeaconBlockBodyElectra { @@ -726,6 +837,7 @@ impl BeaconBlockBodyElectra> { }, bls_to_execution_changes: bls_to_execution_changes.clone(), blob_kzg_commitments: blob_kzg_commitments.clone(), + consolidations: consolidations.clone(), } } } @@ -759,13 +871,31 @@ impl BeaconBlockBody { _ => return Err(Error::IndexNotSupported(generalized_index)), }; + let attestations_root = match self { + BeaconBlockBody::Base(_) + | BeaconBlockBody::Altair(_) + | BeaconBlockBody::Merge(_) + | BeaconBlockBody::Capella(_) + | BeaconBlockBody::Deneb(_) => self.attestations_base()?.tree_hash_root(), + BeaconBlockBody::Electra(_) => self.attestations_electra()?.tree_hash_root(), + }; + + let attester_slashings_root = match self { + BeaconBlockBody::Base(_) + | BeaconBlockBody::Altair(_) + | BeaconBlockBody::Merge(_) + | BeaconBlockBody::Capella(_) + | BeaconBlockBody::Deneb(_) => self.attester_slashings_base()?.tree_hash_root(), + BeaconBlockBody::Electra(_) => self.attester_slashings_electra()?.tree_hash_root(), + }; + let mut leaves = vec![ self.randao_reveal().tree_hash_root(), self.eth1_data().tree_hash_root(), self.graffiti().tree_hash_root(), self.proposer_slashings().tree_hash_root(), - self.attester_slashings().tree_hash_root(), - self.attestations().tree_hash_root(), + attester_slashings_root, + attestations_root, self.deposits().tree_hash_root(), self.voluntary_exits().tree_hash_root(), ]; diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index 4477b236fad..d17d00a3fa8 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -69,15 +69,16 @@ impl IndexedAttestation { /// /// Spec v0.12.1 pub fn is_double_vote(&self, other: &Self) -> bool { - self.data().target.epoch == other.data().target.epoch && self.data() != other.data() + // reuse the ref implementation to ensure logic is the same + self.to_ref().is_double_vote(other.to_ref()) } /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. /// /// Spec v0.12.1 pub fn is_surround_vote(&self, other: &Self) -> bool { - self.data().source.epoch < other.data().source.epoch - && other.data().target.epoch < self.data().target.epoch + // reuse the ref implementation to ensure logic is the same + self.to_ref().is_surround_vote(other.to_ref()) } pub fn attesting_indices_len(&self) -> usize { @@ -116,6 +117,59 @@ impl IndexedAttestation { } } +impl<'a, E: EthSpec> IndexedAttestationRef<'a, E> { + pub fn is_double_vote(&self, other: Self) -> bool { + self.data().target.epoch == other.data().target.epoch && self.data() != other.data() + } + + pub fn is_surround_vote(&self, other: Self) -> bool { + self.data().source.epoch < other.data().source.epoch + && other.data().target.epoch < self.data().target.epoch + } + + pub fn attesting_indices_len(&self) -> usize { + match self { + IndexedAttestationRef::Base(att) => att.attesting_indices.len(), + IndexedAttestationRef::Electra(att) => att.attesting_indices.len(), + } + } + + pub fn attesting_indices_to_vec(&self) -> Vec { + match self { + IndexedAttestationRef::Base(att) => att.attesting_indices.to_vec(), + IndexedAttestationRef::Electra(att) => att.attesting_indices.to_vec(), + } + } + + pub fn attesting_indices_is_empty(&self) -> bool { + match self { + IndexedAttestationRef::Base(att) => att.attesting_indices.is_empty(), + IndexedAttestationRef::Electra(att) => att.attesting_indices.is_empty(), + } + } + + pub fn attesting_indices_iter(&self) -> Iter<'_, u64> { + match self { + IndexedAttestationRef::Base(att) => att.attesting_indices.iter(), + IndexedAttestationRef::Electra(att) => att.attesting_indices.iter(), + } + } + + pub fn attesting_indices_first(&self) -> Option<&u64> { + match self { + IndexedAttestationRef::Base(att) => att.attesting_indices.first(), + IndexedAttestationRef::Electra(att) => att.attesting_indices.first(), + } + } + + pub fn clone_as_indexed_attestation(self) -> IndexedAttestation { + match self { + IndexedAttestationRef::Base(att) => IndexedAttestation::Base(att.clone()), + IndexedAttestationRef::Electra(att) => IndexedAttestation::Electra(att.clone()), + } + } +} + impl Decode for IndexedAttestation { fn is_ssz_fixed_len() -> bool { false diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index feefbc48946..0349e776943 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -115,10 +115,16 @@ use ethereum_types::{H160, H256}; pub use crate::activation_queue::ActivationQueue; pub use crate::aggregate_and_proof::AggregateAndProof; -pub use crate::attestation::{Attestation, Error as AttestationError}; +pub use crate::attestation::{ + Attestation, AttestationBase, AttestationElectra, AttestationRef, AttestationRefMut, + Error as AttestationError, +}; pub use crate::attestation_data::AttestationData; pub use crate::attestation_duty::AttestationDuty; -pub use crate::attester_slashing::AttesterSlashing; +pub use crate::attester_slashing::{ + AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, AttesterSlashingOnDisk, + AttesterSlashingRef, AttesterSlashingRefOnDisk, +}; pub use crate::beacon_block::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockDeneb, BeaconBlockElectra, BeaconBlockMerge, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, @@ -169,7 +175,9 @@ pub use crate::fork_name::{ForkName, InconsistentFork}; pub use crate::fork_versioned_response::{ForkVersionDeserialize, ForkVersionedResponse}; pub use crate::graffiti::{Graffiti, GRAFFITI_BYTES_LEN}; pub use crate::historical_batch::HistoricalBatch; -pub use crate::indexed_attestation::IndexedAttestation; +pub use crate::indexed_attestation::{ + IndexedAttestation, IndexedAttestationBase, IndexedAttestationElectra, IndexedAttestationRef, +}; pub use crate::light_client_bootstrap::{ LightClientBootstrap, LightClientBootstrapAltair, LightClientBootstrapCapella, LightClientBootstrapDeneb, diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index da4ac392362..6c12277700a 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -498,6 +498,7 @@ impl SignedBeaconBlockElectra> { execution_payload: BlindedPayloadElectra { .. }, bls_to_execution_changes, blob_kzg_commitments, + consolidations, }, }, signature, @@ -521,6 +522,7 @@ impl SignedBeaconBlockElectra> { execution_payload: FullPayloadElectra { execution_payload }, bls_to_execution_changes, blob_kzg_commitments, + consolidations, }, }, signature, diff --git a/consensus/types/src/sync_committee_contribution.rs b/consensus/types/src/sync_committee_contribution.rs index 2c20daa9702..81de83ba3be 100644 --- a/consensus/types/src/sync_committee_contribution.rs +++ b/consensus/types/src/sync_committee_contribution.rs @@ -110,6 +110,12 @@ impl SlotData for SyncCommitteeContribution { } } +impl SlotData for &SyncCommitteeContribution { + fn get_slot(&self) -> Slot { + self.slot + } +} + impl SlotData for SyncContributionData { fn get_slot(&self) -> Slot { self.slot diff --git a/lcli/src/indexed_attestations.rs b/lcli/src/indexed_attestations.rs index 4c8b06a5088..8d932ba7600 100644 --- a/lcli/src/indexed_attestations.rs +++ b/lcli/src/indexed_attestations.rs @@ -35,7 +35,7 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { .into_iter() .map(|att| { let committee = state.get_beacon_committee(att.data().slot, att.data().index)?; - get_indexed_attestation(committee.committee, &att) + get_indexed_attestation(committee.committee, att.to_ref()) }) .collect::, _>>() .map_err(|e| format!("Error constructing indexed attestation: {:?}", e))?; diff --git a/lcli/src/transition_blocks.rs b/lcli/src/transition_blocks.rs index 77fd352829f..5c7231a9eda 100644 --- a/lcli/src/transition_blocks.rs +++ b/lcli/src/transition_blocks.rs @@ -390,7 +390,7 @@ fn do_transition( // Signature verification should prime the indexed attestation cache. assert_eq!( ctxt.num_cached_indexed_attestations(), - block.message().body().attestations().len() + block.message().body().attestations_len() ); } diff --git a/slasher/src/lib.rs b/slasher/src/lib.rs index 45cbef84f21..2577829577e 100644 --- a/slasher/src/lib.rs +++ b/slasher/src/lib.rs @@ -28,7 +28,8 @@ pub use database::{ }; pub use error::Error; -use types::{AttesterSlashing, EthSpec, IndexedAttestation, ProposerSlashing}; +use types::{AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra}; +use types::{EthSpec, IndexedAttestation, ProposerSlashing}; #[derive(Debug, PartialEq)] pub enum AttesterSlashingStatus { @@ -59,14 +60,48 @@ impl AttesterSlashingStatus { match self { NotSlashable => None, AlreadyDoubleVoted => None, - DoubleVote(existing) | SurroundedByExisting(existing) => Some(AttesterSlashing { - attestation_1: *existing, - attestation_2: new_attestation.clone(), - }), - SurroundsExisting(existing) => Some(AttesterSlashing { + DoubleVote(existing) | SurroundedByExisting(existing) => { + match (*existing, new_attestation) { + // TODO(electra) - determine when we would convert a Base attestation to Electra / how to handle mismatched attestations here + (IndexedAttestation::Base(existing_att), IndexedAttestation::Base(new_att)) => { + Some(AttesterSlashing::Base(AttesterSlashingBase { + attestation_1: existing_att, + attestation_2: new_att.clone(), + })) + } + ( + IndexedAttestation::Electra(existing_att), + IndexedAttestation::Electra(new_att), + ) => Some(AttesterSlashing::Electra(AttesterSlashingElectra { + attestation_1: existing_att, + attestation_2: new_att.clone(), + })), + _ => panic!("attestations must be of the same type"), + } + } + // TODO(electra): fix this once we superstruct IndexedAttestation (return the correct type) + SurroundsExisting(existing) => match (*existing, new_attestation) { + (IndexedAttestation::Base(existing_att), IndexedAttestation::Base(new_att)) => { + Some(AttesterSlashing::Base(AttesterSlashingBase { + attestation_1: new_att.clone(), + attestation_2: existing_att, + })) + } + ( + IndexedAttestation::Electra(existing_att), + IndexedAttestation::Electra(new_att), + ) => Some(AttesterSlashing::Electra(AttesterSlashingElectra { + attestation_1: new_att.clone(), + attestation_2: existing_att, + })), + _ => panic!("attestations must be of the same type"), + }, + /* + Some(AttesterSlashing::Base(AttesterSlashingBase { attestation_1: new_attestation.clone(), attestation_2: *existing, - }), + })), + */ } } } diff --git a/slasher/src/slasher.rs b/slasher/src/slasher.rs index 40b49fbbc66..fc8e8453c89 100644 --- a/slasher/src/slasher.rs +++ b/slasher/src/slasher.rs @@ -299,7 +299,7 @@ impl Slasher { self.log, "Found double-vote slashing"; "validator_index" => validator_index, - "epoch" => slashing.attestation_1.data().target.epoch, + "epoch" => slashing.attestation_1().data().target.epoch, ); slashings.insert(slashing); } diff --git a/slasher/src/test_utils.rs b/slasher/src/test_utils.rs index c46b2f39492..634b4d52113 100644 --- a/slasher/src/test_utils.rs +++ b/slasher/src/test_utils.rs @@ -1,8 +1,8 @@ use std::collections::HashSet; use types::{ indexed_attestation::IndexedAttestationBase, AggregateSignature, AttestationData, - AttesterSlashing, BeaconBlockHeader, Checkpoint, Epoch, Hash256, IndexedAttestation, - MainnetEthSpec, Signature, SignedBeaconBlockHeader, Slot, + AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BeaconBlockHeader, Checkpoint, + Epoch, Hash256, IndexedAttestation, MainnetEthSpec, Signature, SignedBeaconBlockHeader, Slot, }; pub type E = MainnetEthSpec; @@ -37,9 +37,21 @@ pub fn att_slashing( attestation_1: &IndexedAttestation, attestation_2: &IndexedAttestation, ) -> AttesterSlashing { - AttesterSlashing { - attestation_1: attestation_1.clone(), - attestation_2: attestation_2.clone(), + // TODO(electra): fix this one we superstruct IndexedAttestation (return the correct type) + match (attestation_1, attestation_2) { + (IndexedAttestation::Base(att1), IndexedAttestation::Base(att2)) => { + AttesterSlashing::Base(AttesterSlashingBase { + attestation_1: att1.clone(), + attestation_2: att2.clone(), + }) + } + (IndexedAttestation::Electra(att1), IndexedAttestation::Electra(att2)) => { + AttesterSlashing::Electra(AttesterSlashingElectra { + attestation_1: att1.clone(), + attestation_2: att2.clone(), + }) + } + _ => panic!("attestations must be of the same type"), } } @@ -61,8 +73,8 @@ pub fn slashed_validators_from_slashings(slashings: &HashSet slashings .iter() .flat_map(|slashing| { - let att1 = &slashing.attestation_1; - let att2 = &slashing.attestation_2; + let att1 = slashing.attestation_1(); + let att2 = slashing.attestation_2(); assert!( att1.is_double_vote(att2) || att1.is_surround_vote(att2), "invalid slashing: {:#?}", diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index f0749c3c7e4..bcc1c3aa25b 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -23,8 +23,9 @@ use state_processing::state_advance::complete_state_advance; use std::future::Future; use std::sync::Arc; use std::time::Duration; +use types::AttesterSlashingBase; use types::{ - Attestation, AttesterSlashing, BeaconBlock, BeaconState, BlobSidecar, BlobsList, Checkpoint, + Attestation, AttesterSlashingRef, BeaconBlock, BeaconState, BlobSidecar, BlobsList, Checkpoint, ExecutionBlockHash, Hash256, IndexedAttestation, KzgProof, ProposerPreparationData, SignedBeaconBlock, Slot, Uint256, }; @@ -131,8 +132,9 @@ pub struct ForkChoiceTest { pub anchor_state: BeaconState, pub anchor_block: BeaconBlock, #[allow(clippy::type_complexity)] + // TODO(electra): these tests will need to be updated to use new types pub steps: Vec< - Step, BlobsList, Attestation, AttesterSlashing, PowBlock>, + Step, BlobsList, Attestation, AttesterSlashingBase, PowBlock>, >, } @@ -249,7 +251,7 @@ impl Case for ForkChoiceTest { } => tester.process_block(block.clone(), blobs.clone(), proofs.clone(), *valid)?, Step::Attestation { attestation } => tester.process_attestation(attestation)?, Step::AttesterSlashing { attester_slashing } => { - tester.process_attester_slashing(attester_slashing) + tester.process_attester_slashing(AttesterSlashingRef::Base(attester_slashing)) } Step::PowBlock { pow_block } => tester.process_pow_block(pow_block), Step::OnPayloadInfo { @@ -591,7 +593,7 @@ impl Tester { .map_err(|e| Error::InternalError(format!("attestation import failed with {:?}", e))) } - pub fn process_attester_slashing(&self, attester_slashing: &AttesterSlashing) { + pub fn process_attester_slashing(&self, attester_slashing: AttesterSlashingRef) { self.harness .chain .canonical_head diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index a2f50896a57..15de13610fe 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -93,7 +93,7 @@ impl Operation for Attestation { match state { BeaconState::Base(_) => base::process_attestations( state, - &[self.clone()], + [self.clone().to_ref()].into_iter(), VerifySignatures::True, &mut ctxt, spec, @@ -106,7 +106,7 @@ impl Operation for Attestation { initialize_progressive_balances_cache(state, spec)?; altair_deneb::process_attestation( state, - self, + self.to_ref(), 0, &mut ctxt, VerifySignatures::True, @@ -123,7 +123,7 @@ impl Operation for AttesterSlashing { } fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { - ssz_decode_file(path) + Ok(Self::Base(ssz_decode_file(path)?)) } fn apply_to( @@ -136,7 +136,7 @@ impl Operation for AttesterSlashing { initialize_progressive_balances_cache(state, spec)?; process_attester_slashings( state, - &[self.clone()], + [self.clone().to_ref()].into_iter(), VerifySignatures::True, &mut ctxt, spec, diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 06d484a52bb..cb3636135c1 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -919,8 +919,8 @@ impl SignedBlock { } pub fn num_attestations(&self) -> usize { match self { - SignedBlock::Full(block) => block.signed_block().message().body().attestations().len(), - SignedBlock::Blinded(block) => block.message().body().attestations().len(), + SignedBlock::Full(block) => block.signed_block().message().body().attestations_len(), + SignedBlock::Blinded(block) => block.message().body().attestations_len(), } } } diff --git a/watch/src/database/mod.rs b/watch/src/database/mod.rs index 59547c303ab..7d8c4c68cc2 100644 --- a/watch/src/database/mod.rs +++ b/watch/src/database/mod.rs @@ -141,7 +141,7 @@ pub fn insert_beacon_block( let parent_root = WatchHash::from_hash(block.parent_root()); let proposer_index = block_message.proposer_index() as i32; let graffiti = block_message.body().graffiti().as_utf8_lossy(); - let attestation_count = block_message.body().attestations().len() as i32; + let attestation_count = block_message.body().attestations_len() as i32; let full_payload = block_message.execution_payload().ok(); From 9b98f4e297358d454e2bb540ffe51a2a170ba6f7 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Fri, 3 May 2024 13:57:01 -0500 Subject: [PATCH 003/227] Make EF Tests Fork-Agnostic (#5713) --- testing/ef_tests/src/cases/fork_choice.rs | 33 ++++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index bcc1c3aa25b..7b60e331666 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -23,11 +23,10 @@ use state_processing::state_advance::complete_state_advance; use std::future::Future; use std::sync::Arc; use std::time::Duration; -use types::AttesterSlashingBase; use types::{ - Attestation, AttesterSlashingRef, BeaconBlock, BeaconState, BlobSidecar, BlobsList, Checkpoint, - ExecutionBlockHash, Hash256, IndexedAttestation, KzgProof, ProposerPreparationData, - SignedBeaconBlock, Slot, Uint256, + Attestation, AttesterSlashing, AttesterSlashingRef, BeaconBlock, BeaconState, BlobSidecar, + BlobsList, Checkpoint, ExecutionBlockHash, Hash256, IndexedAttestation, KzgProof, + ProposerPreparationData, SignedBeaconBlock, Slot, Uint256, }; #[derive(Default, Debug, PartialEq, Clone, Deserialize, Decode)] @@ -134,7 +133,7 @@ pub struct ForkChoiceTest { #[allow(clippy::type_complexity)] // TODO(electra): these tests will need to be updated to use new types pub steps: Vec< - Step, BlobsList, Attestation, AttesterSlashingBase, PowBlock>, + Step, BlobsList, Attestation, AttesterSlashing, PowBlock>, >, } @@ -185,10 +184,24 @@ impl LoadCase for ForkChoiceTest { ssz_decode_file(&path.join(format!("{}.ssz_snappy", attestation))) .map(|attestation| Step::Attestation { attestation }) } - Step::AttesterSlashing { attester_slashing } => { - ssz_decode_file(&path.join(format!("{}.ssz_snappy", attester_slashing))) - .map(|attester_slashing| Step::AttesterSlashing { attester_slashing }) - } + Step::AttesterSlashing { attester_slashing } => match fork_name { + ForkName::Base + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => { + ssz_decode_file(&path.join(format!("{}.ssz_snappy", attester_slashing))) + .map(|attester_slashing| Step::AttesterSlashing { + attester_slashing: AttesterSlashing::Base(attester_slashing), + }) + } + ForkName::Electra => { + ssz_decode_file(&path.join(format!("{}.ssz_snappy", attester_slashing))) + .map(|attester_slashing| Step::AttesterSlashing { + attester_slashing: AttesterSlashing::Electra(attester_slashing), + }) + } + }, Step::PowBlock { pow_block } => { ssz_decode_file(&path.join(format!("{}.ssz_snappy", pow_block))) .map(|pow_block| Step::PowBlock { pow_block }) @@ -251,7 +264,7 @@ impl Case for ForkChoiceTest { } => tester.process_block(block.clone(), blobs.clone(), proofs.clone(), *valid)?, Step::Attestation { attestation } => tester.process_attestation(attestation)?, Step::AttesterSlashing { attester_slashing } => { - tester.process_attester_slashing(AttesterSlashingRef::Base(attester_slashing)) + tester.process_attester_slashing(attester_slashing.to_ref()) } Step::PowBlock { pow_block } => tester.process_pow_block(pow_block), Step::OnPayloadInfo { From 7c6526d9782c5f61bcd7cc79f7120b4877d56c19 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Fri, 3 May 2024 14:09:49 -0500 Subject: [PATCH 004/227] Finish EF Test Fork Agnostic (#5714) --- beacon_node/lighthouse_network/src/types/pubsub.rs | 1 - testing/ef_tests/src/cases/operations.rs | 11 +++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 4c7e115aed7..4af0f54f31e 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -239,7 +239,6 @@ impl PubsubMessage { Ok(PubsubMessage::ProposerSlashing(Box::new(proposer_slashing))) } GossipKind::AttesterSlashing => { - // TODO(electra): could an older attester slashing still be floating around during the fork transition? let attester_slashing = match fork_context.from_context_bytes(gossip_topic.fork_digest) { Some(ForkName::Base) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index d76620bd22d..b922668c0a0 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -122,8 +122,15 @@ impl Operation for AttesterSlashing { "attester_slashing".into() } - fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { - Ok(Self::Base(ssz_decode_file(path)?)) + fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { + Ok(match fork_name { + ForkName::Base + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => Self::Base(ssz_decode_file(path)?), + ForkName::Electra => Self::Electra(ssz_decode_file(path)?), + }) } fn apply_to( From 19a94792348b6926c47143e572e6380927ddd5fc Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Mon, 6 May 2024 10:09:22 -0500 Subject: [PATCH 005/227] Superstruct `AggregateAndProof` (#5715) * Upgrade `superstruct` to `0.8.0` * superstruct `AggregateAndProof` --- Cargo.lock | 5 +- Cargo.toml | 2 +- .../src/attestation_verification.rs | 82 +++++++++-------- beacon_node/beacon_chain/src/beacon_chain.rs | 10 +- .../beacon_chain/src/bellatrix_readiness.rs | 4 +- .../src/naive_aggregation_pool.rs | 79 ++++++++++------ beacon_node/beacon_chain/src/test_utils.rs | 2 +- .../beacon_chain/src/validator_monitor.rs | 2 +- beacon_node/execution_layer/src/lib.rs | 2 +- beacon_node/http_api/src/lib.rs | 8 +- .../http_api/src/publish_attestations.rs | 2 +- beacon_node/http_api/tests/tests.rs | 10 +- .../lighthouse_network/src/types/pubsub.rs | 34 +++++-- .../gossip_methods.rs | 29 +++--- .../src/network_beacon_processor/mod.rs | 2 +- beacon_node/operation_pool/src/lib.rs | 25 ++--- .../per_block_processing/signature_sets.rs | 16 ++-- consensus/types/src/aggregate_and_proof.rs | 92 ++++++++++++++----- consensus/types/src/attestation.rs | 49 ++++++---- consensus/types/src/attester_slashing.rs | 13 +-- .../types/src/execution_payload_header.rs | 9 +- consensus/types/src/lib.rs | 8 +- .../types/src/signed_aggregate_and_proof.rs | 71 ++++++++++---- testing/ef_tests/src/cases/fork_choice.rs | 18 ++-- validator_client/src/attestation_service.rs | 8 +- validator_client/src/signing_method.rs | 2 +- .../src/signing_method/web3signer.rs | 2 +- validator_client/src/validator_store.rs | 45 ++++++--- 28 files changed, 408 insertions(+), 223 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a52b854ced..15c2e1280ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8028,9 +8028,8 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "superstruct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f4e1f478a7728f8855d7e620e9a152cf8932c6614f86564c886f9b8141f3201" +version = "0.8.0" +source = "git+https://github.com/sigp/superstruct?rev=45eecbfb9708c9fe11dbb6a6a5bd8d618f02269e#45eecbfb9708c9fe11dbb6a6a5bd8d618f02269e" dependencies = [ "darling", "itertools", diff --git a/Cargo.toml b/Cargo.toml index c70d9459124..09e2b7c4346 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -162,7 +162,7 @@ smallvec = "1.11.2" snap = "1" ssz_types = "0.6" strum = { version = "0.24", features = ["derive"] } -superstruct = "0.7" +superstruct = { git = "https://github.com/sigp/superstruct", rev = "45eecbfb9708c9fe11dbb6a6a5bd8d618f02269e" } syn = "1" sysinfo = "0.26" tempfile = "3" diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 1dd43b3c024..dff3fd6f2dd 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -55,8 +55,8 @@ use std::borrow::Cow; use strum::AsRefStr; use tree_hash::TreeHash; use types::{ - Attestation, BeaconCommittee, ChainSpec, CommitteeIndex, Epoch, EthSpec, ForkName, Hash256, - IndexedAttestation, SelectionProof, SignedAggregateAndProof, Slot, SubnetId, + Attestation, AttestationRef, BeaconCommittee, ChainSpec, CommitteeIndex, Epoch, EthSpec, + ForkName, Hash256, IndexedAttestation, SelectionProof, SignedAggregateAndProof, Slot, SubnetId, }; pub use batch::{batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations}; @@ -274,7 +274,7 @@ struct IndexedAggregatedAttestation<'a, T: BeaconChainTypes> { /// /// These attestations have *not* undergone signature verification. struct IndexedUnaggregatedAttestation<'a, T: BeaconChainTypes> { - attestation: &'a Attestation, + attestation: AttestationRef<'a, T::EthSpec>, indexed_attestation: IndexedAttestation, subnet_id: SubnetId, validator_index: u64, @@ -295,7 +295,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { /// Wraps an `Attestation` that has been fully verified for propagation on the gossip network. pub struct VerifiedUnaggregatedAttestation<'a, T: BeaconChainTypes> { - attestation: &'a Attestation, + attestation: AttestationRef<'a, T::EthSpec>, indexed_attestation: IndexedAttestation, subnet_id: SubnetId, } @@ -322,20 +322,20 @@ impl<'a, T: BeaconChainTypes> Clone for IndexedUnaggregatedAttestation<'a, T> { /// A helper trait implemented on wrapper types that can be progressed to a state where they can be /// verified for application to fork choice. pub trait VerifiedAttestation: Sized { - fn attestation(&self) -> &Attestation; + fn attestation(&self) -> AttestationRef; fn indexed_attestation(&self) -> &IndexedAttestation; // Inefficient default implementation. This is overridden for gossip verified attestations. fn into_attestation_and_indices(self) -> (Attestation, Vec) { - let attestation = self.attestation().clone(); + let attestation = self.attestation().clone_as_attestation(); let attesting_indices = self.indexed_attestation().attesting_indices_to_vec(); (attestation, attesting_indices) } } impl<'a, T: BeaconChainTypes> VerifiedAttestation for VerifiedAggregatedAttestation<'a, T> { - fn attestation(&self) -> &Attestation { + fn attestation(&self) -> AttestationRef { self.attestation() } @@ -345,7 +345,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAttestation for VerifiedAggregatedAttes } impl<'a, T: BeaconChainTypes> VerifiedAttestation for VerifiedUnaggregatedAttestation<'a, T> { - fn attestation(&self) -> &Attestation { + fn attestation(&self) -> AttestationRef { self.attestation } @@ -357,7 +357,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAttestation for VerifiedUnaggregatedAtt /// Information about invalid attestations which might still be slashable despite being invalid. pub enum AttestationSlashInfo<'a, T: BeaconChainTypes, TErr> { /// The attestation is invalid, but its signature wasn't checked. - SignatureNotChecked(&'a Attestation, TErr), + SignatureNotChecked(AttestationRef<'a, T::EthSpec>, TErr), /// As for `SignatureNotChecked`, but we know the `IndexedAttestation`. SignatureNotCheckedIndexed(IndexedAttestation, TErr), /// The attestation's signature is invalid, so it will never be slashable. @@ -446,7 +446,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { signed_aggregate: &SignedAggregateAndProof, chain: &BeaconChain, ) -> Result { - let attestation = &signed_aggregate.message.aggregate; + let attestation = signed_aggregate.message().aggregate(); // Ensure attestation is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (within a // MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance). @@ -471,14 +471,14 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { if chain .observed_attestations .write() - .is_known_subset(attestation.to_ref(), attestation_data_root) + .is_known_subset(attestation, attestation_data_root) .map_err(|e| Error::BeaconChainError(e.into()))? { metrics::inc_counter(&metrics::AGGREGATED_ATTESTATION_SUBSETS); return Err(Error::AttestationSupersetKnown(attestation_data_root)); } - let aggregator_index = signed_aggregate.message.aggregator_index; + let aggregator_index = signed_aggregate.message().aggregator_index(); // Ensure there has been no other observed aggregate for the given `aggregator_index`. // @@ -532,11 +532,16 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { ) -> Result> { use AttestationSlashInfo::*; - let attestation = &signed_aggregate.message.aggregate; - let aggregator_index = signed_aggregate.message.aggregator_index; + let attestation = signed_aggregate.message().aggregate(); + let aggregator_index = signed_aggregate.message().aggregator_index(); let attestation_data_root = match Self::verify_early_checks(signed_aggregate, chain) { Ok(root) => root, - Err(e) => return Err(SignatureNotChecked(&signed_aggregate.message.aggregate, e)), + Err(e) => { + return Err(SignatureNotChecked( + signed_aggregate.message().aggregate(), + e, + )) + } }; let get_indexed_attestation_with_committee = @@ -545,7 +550,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { // // Future optimizations should remove this clone. let selection_proof = - SelectionProof::from(signed_aggregate.message.selection_proof.clone()); + SelectionProof::from(signed_aggregate.message().selection_proof().clone()); if !selection_proof .is_aggregator(committee.committee.len(), &chain.spec) @@ -559,7 +564,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { return Err(Error::AggregatorNotInCommittee { aggregator_index }); } - get_indexed_attestation(committee.committee, attestation.to_ref()) + get_indexed_attestation(committee.committee, attestation) .map_err(|e| BeaconChainError::from(e).into()) }; @@ -569,7 +574,12 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { get_indexed_attestation_with_committee, ) { Ok(indexed_attestation) => indexed_attestation, - Err(e) => return Err(SignatureNotChecked(&signed_aggregate.message.aggregate, e)), + Err(e) => { + return Err(SignatureNotChecked( + signed_aggregate.message().aggregate(), + e, + )) + } }; Ok(IndexedAggregatedAttestation { @@ -587,8 +597,8 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { attestation_data_root: Hash256, chain: &BeaconChain, ) -> Result<(), Error> { - let attestation = &signed_aggregate.message.aggregate; - let aggregator_index = signed_aggregate.message.aggregator_index; + let attestation = signed_aggregate.message().aggregate(); + let aggregator_index = signed_aggregate.message().aggregator_index(); // Observe the valid attestation so we do not re-process it. // @@ -597,7 +607,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { if let ObserveOutcome::Subset = chain .observed_attestations .write() - .observe_item(attestation.to_ref(), Some(attestation_data_root)) + .observe_item(attestation, Some(attestation_data_root)) .map_err(|e| Error::BeaconChainError(e.into()))? { metrics::inc_counter(&metrics::AGGREGATED_ATTESTATION_SUBSETS); @@ -696,8 +706,8 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { } /// Returns the underlying `attestation` for the `signed_aggregate`. - pub fn attestation(&self) -> &Attestation { - &self.signed_aggregate.message.aggregate + pub fn attestation(&self) -> AttestationRef<'a, T::EthSpec> { + self.signed_aggregate.message().aggregate() } /// Returns the underlying `signed_aggregate`. @@ -709,7 +719,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { /// Run the checks that happen before an indexed attestation is constructed. pub fn verify_early_checks( - attestation: &Attestation, + attestation: AttestationRef, chain: &BeaconChain, ) -> Result<(), Error> { let attestation_epoch = attestation.data().slot.epoch(T::EthSpec::slots_per_epoch()); @@ -750,7 +760,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { /// Run the checks that apply to the indexed attestation before the signature is checked. pub fn verify_middle_checks( - attestation: &Attestation, + attestation: AttestationRef, indexed_attestation: &IndexedAttestation, committees_per_slot: u64, subnet_id: Option, @@ -806,7 +816,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { subnet_id: Option, chain: &BeaconChain, ) -> Result { - Self::verify_slashable(attestation, subnet_id, chain) + Self::verify_slashable(attestation.to_ref(), subnet_id, chain) .map(|verified_unaggregated| { if let Some(slasher) = chain.slasher.as_ref() { slasher.accept_attestation(verified_unaggregated.indexed_attestation.clone()); @@ -818,7 +828,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { /// Verify the attestation, producing extra information about whether it might be slashable. pub fn verify_slashable( - attestation: &'a Attestation, + attestation: AttestationRef<'a, T::EthSpec>, subnet_id: Option, chain: &BeaconChain, ) -> Result> { @@ -867,7 +877,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { impl<'a, T: BeaconChainTypes> VerifiedUnaggregatedAttestation<'a, T> { /// Run the checks that apply after the signature has been checked. fn verify_late_checks( - attestation: &Attestation, + attestation: AttestationRef, validator_index: u64, chain: &BeaconChain, ) -> Result<(), Error> { @@ -961,7 +971,7 @@ impl<'a, T: BeaconChainTypes> VerifiedUnaggregatedAttestation<'a, T> { } /// Returns the wrapped `attestation`. - pub fn attestation(&self) -> &Attestation { + pub fn attestation(&self) -> AttestationRef { self.attestation } @@ -991,7 +1001,7 @@ impl<'a, T: BeaconChainTypes> VerifiedUnaggregatedAttestation<'a, T> { /// already finalized. fn verify_head_block_is_known( chain: &BeaconChain, - attestation: &Attestation, + attestation: AttestationRef, max_skip_slots: Option, ) -> Result { let block_opt = chain @@ -1039,7 +1049,7 @@ fn verify_head_block_is_known( /// Accounts for `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. pub fn verify_propagation_slot_range( slot_clock: &S, - attestation: &Attestation, + attestation: AttestationRef, spec: &ChainSpec, ) -> Result<(), Error> { let attestation_slot = attestation.data().slot; @@ -1124,7 +1134,7 @@ pub fn verify_attestation_signature( /// `attestation.data.beacon_block_root`. pub fn verify_attestation_target_root( head_block: &ProtoBlock, - attestation: &Attestation, + attestation: AttestationRef, ) -> Result<(), Error> { // Check the attestation target root. let head_block_epoch = head_block.slot.epoch(E::slots_per_epoch()); @@ -1193,7 +1203,7 @@ pub fn verify_signed_aggregate_signatures( .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT) .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)?; - let aggregator_index = signed_aggregate.message.aggregator_index; + let aggregator_index = signed_aggregate.message().aggregator_index(); if aggregator_index >= pubkey_cache.len() as u64 { return Err(Error::AggregatorPubkeyUnknown(aggregator_index)); } @@ -1240,10 +1250,10 @@ type CommitteesPerSlot = u64; /// public keys cached in the `chain`. pub fn obtain_indexed_attestation_and_committees_per_slot( chain: &BeaconChain, - attestation: &Attestation, + attestation: AttestationRef, ) -> Result<(IndexedAttestation, CommitteesPerSlot), Error> { map_attestation_committee(chain, attestation, |(committee, committees_per_slot)| { - get_indexed_attestation(committee.committee, attestation.to_ref()) + get_indexed_attestation(committee.committee, attestation) .map(|attestation| (attestation, committees_per_slot)) .map_err(Error::Invalid) }) @@ -1260,7 +1270,7 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( /// from disk and then update the `shuffling_cache`. fn map_attestation_committee( chain: &BeaconChain, - attestation: &Attestation, + attestation: AttestationRef, map_fn: F, ) -> Result where diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a99b5db6051..cd47068977b 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1966,8 +1966,9 @@ impl BeaconChain { // This method is called for API and gossip attestations, so this covers all unaggregated attestation events if let Some(event_handler) = self.event_handler.as_ref() { if event_handler.has_attestation_subscribers() { - event_handler - .register(EventKind::Attestation(Box::new(v.attestation().clone()))); + event_handler.register(EventKind::Attestation(Box::new( + v.attestation().clone_as_attestation(), + ))); } } metrics::inc_counter(&metrics::UNAGGREGATED_ATTESTATION_PROCESSING_SUCCESSES); @@ -2003,8 +2004,9 @@ impl BeaconChain { // This method is called for API and gossip attestations, so this covers all aggregated attestation events if let Some(event_handler) = self.event_handler.as_ref() { if event_handler.has_attestation_subscribers() { - event_handler - .register(EventKind::Attestation(Box::new(v.attestation().clone()))); + event_handler.register(EventKind::Attestation(Box::new( + v.attestation().clone_as_attestation(), + ))); } } metrics::inc_counter(&metrics::AGGREGATED_ATTESTATION_PROCESSING_SUCCESSES); diff --git a/beacon_node/beacon_chain/src/bellatrix_readiness.rs b/beacon_node/beacon_chain/src/bellatrix_readiness.rs index bf9e8481261..60b1abaf098 100644 --- a/beacon_node/beacon_chain/src/bellatrix_readiness.rs +++ b/beacon_node/beacon_chain/src/bellatrix_readiness.rs @@ -244,8 +244,8 @@ impl BeaconChain { }); } - if let Some(&expected) = expected_withdrawals_root { - if let Some(&got) = got_withdrawals_root { + if let Some(expected) = expected_withdrawals_root { + if let Some(got) = got_withdrawals_root { if got != expected { return Ok(GenesisExecutionPayloadStatus::WithdrawalsRootMismatch { got, diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index 0eab13a4c1d..a12521cd171 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -1,10 +1,13 @@ use crate::metrics; +use crate::observed_aggregates::AsReference; use std::collections::HashMap; use tree_hash::TreeHash; use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT; use types::slot_data::SlotData; use types::sync_committee_contribution::SyncContributionData; -use types::{Attestation, AttestationData, EthSpec, Hash256, Slot, SyncCommitteeContribution}; +use types::{ + Attestation, AttestationData, AttestationRef, EthSpec, Hash256, Slot, SyncCommitteeContribution, +}; type AttestationDataRoot = Hash256; type SyncDataRoot = Hash256; @@ -59,12 +62,15 @@ pub enum Error { /// Implemented for items in the `NaiveAggregationPool`. Requires that items implement `SlotData`, /// which means they have an associated slot. This handles aggregation of items that are inserted. -pub trait AggregateMap { +pub trait AggregateMap +where + for<'a> ::Reference<'a>: SlotData, +{ /// `Key` should be a hash of `Data`. type Key; /// The item stored in the map - type Value: Clone + SlotData; + type Value: Clone + SlotData + AsReference; /// The unique fields of `Value`, hashed to create `Key`. type Data: SlotData; @@ -73,7 +79,10 @@ pub trait AggregateMap { fn new(initial_capacity: usize) -> Self; /// Insert a `Value` into `Self`, returning a result. - fn insert(&mut self, value: &Self::Value) -> Result; + fn insert( + &mut self, + value: ::Reference<'_>, + ) -> Result; /// Get a `Value` from `Self` based on `Data`. fn get(&self, data: &Self::Data) -> Option; @@ -121,18 +130,18 @@ impl AggregateMap for AggregatedAttestationMap { /// Insert an attestation into `self`, aggregating it into the pool. /// /// The given attestation (`a`) must only have one signature. - fn insert(&mut self, a: &Self::Value) -> Result { + fn insert(&mut self, a: AttestationRef) -> Result { let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_CORE_INSERT); let set_bits = match a { - Attestation::Base(att) => att + AttestationRef::Base(att) => att .aggregation_bits .iter() .enumerate() .filter(|(_i, bit)| *bit) .map(|(i, _bit)| i) .collect::>(), - Attestation::Electra(att) => att + AttestationRef::Electra(att) => att .aggregation_bits .iter() .enumerate() @@ -169,7 +178,8 @@ impl AggregateMap for AggregatedAttestationMap { return Err(Error::ReachedMaxItemsPerSlot(MAX_ATTESTATIONS_PER_SLOT)); } - self.map.insert(attestation_data_root, a.clone()); + self.map + .insert(attestation_data_root, a.clone_as_attestation()); Ok(InsertOutcome::NewItemInserted { committee_index }) } } @@ -344,12 +354,20 @@ impl AggregateMap for SyncContributionAggregateMap { /// `current_slot - SLOTS_RETAINED` will be removed and any future item with a slot lower /// than that will also be refused. Pruning is done automatically based upon the items it /// receives and it can be triggered manually. -pub struct NaiveAggregationPool { +pub struct NaiveAggregationPool +where + T: AggregateMap, + for<'a> ::Reference<'a>: SlotData, +{ lowest_permissible_slot: Slot, maps: HashMap, } -impl Default for NaiveAggregationPool { +impl Default for NaiveAggregationPool +where + T: AggregateMap, + for<'a> ::Reference<'a>: SlotData, +{ fn default() -> Self { Self { lowest_permissible_slot: Slot::new(0), @@ -358,7 +376,11 @@ impl Default for NaiveAggregationPool { } } -impl NaiveAggregationPool { +impl NaiveAggregationPool +where + T: AggregateMap, + for<'a> ::Reference<'a>: SlotData, +{ /// Insert an item into `self`, aggregating it into the pool. /// /// The given item must only have one signature and have an @@ -366,7 +388,10 @@ impl NaiveAggregationPool { /// /// The pool may be pruned if the given item has a slot higher than any /// previously seen. - pub fn insert(&mut self, item: &T::Value) -> Result { + pub fn insert( + &mut self, + item: ::Reference<'_>, + ) -> Result { let _timer = T::start_insert_timer(); let slot = item.get_slot(); let lowest_permissible_slot = self.lowest_permissible_slot; @@ -602,10 +627,10 @@ mod tests { let mut a = $get_method_name(Slot::new(0)); let mut pool: NaiveAggregationPool<$map_type> = - NaiveAggregationPool::default(); + NaiveAggregationPool::<$map_type>::default(); assert_eq!( - pool.insert(&a), + pool.insert(a.as_reference()), Err(Error::NoAggregationBitsSet), "should not accept item without any signatures" ); @@ -613,12 +638,12 @@ mod tests { $sign_method_name(&mut a, 0, Hash256::random()); assert_eq!( - pool.insert(&a), + pool.insert(a.as_reference()), Ok(InsertOutcome::NewItemInserted { committee_index: 0 }), "should accept new item" ); assert_eq!( - pool.insert(&a), + pool.insert(a.as_reference()), Ok(InsertOutcome::SignatureAlreadyKnown { committee_index: 0 }), "should acknowledge duplicate signature" ); @@ -631,7 +656,7 @@ mod tests { $sign_method_name(&mut a, 1, Hash256::random()); assert_eq!( - pool.insert(&a), + pool.insert(a.as_reference()), Err(Error::MoreThanOneAggregationBitSet(2)), "should not accept item with multiple signatures" ); @@ -647,15 +672,15 @@ mod tests { $sign_method_name(&mut a_1, 1, genesis_validators_root); let mut pool: NaiveAggregationPool<$map_type> = - NaiveAggregationPool::default(); + NaiveAggregationPool::<$map_type>::default(); assert_eq!( - pool.insert(&a_0), + pool.insert(a_0.as_reference()), Ok(InsertOutcome::NewItemInserted { committee_index: 0 }), "should accept a_0" ); assert_eq!( - pool.insert(&a_1), + pool.insert(a_1.as_reference()), Ok(InsertOutcome::SignatureAggregated { committee_index: 1 }), "should accept a_1" ); @@ -665,7 +690,7 @@ mod tests { .expect("should not error while getting attestation"); let mut a_01 = a_0.clone(); - a_01.aggregate(&a_1); + a_01.aggregate(a_1.as_reference()); assert_eq!(retrieved, a_01, "retrieved item should be aggregated"); @@ -681,7 +706,7 @@ mod tests { $block_root_mutator(&mut a_different, different_root); assert_eq!( - pool.insert(&a_different), + pool.insert(a_different.as_reference()), Ok(InsertOutcome::NewItemInserted { committee_index: 2 }), "should accept a_different" ); @@ -700,7 +725,7 @@ mod tests { $sign_method_name(&mut base, 0, Hash256::random()); let mut pool: NaiveAggregationPool<$map_type> = - NaiveAggregationPool::default(); + NaiveAggregationPool::<$map_type>::default(); for i in 0..SLOTS_RETAINED * 2 { let slot = Slot::from(i); @@ -708,7 +733,7 @@ mod tests { $slot_mutator(&mut a, slot); assert_eq!( - pool.insert(&a), + pool.insert(a.as_reference()), Ok(InsertOutcome::NewItemInserted { committee_index: 0 }), "should accept new item" ); @@ -749,7 +774,7 @@ mod tests { $sign_method_name(&mut base, 0, Hash256::random()); let mut pool: NaiveAggregationPool<$map_type> = - NaiveAggregationPool::default(); + NaiveAggregationPool::<$map_type>::default(); for i in 0..=$item_limit { let mut a = base.clone(); @@ -757,13 +782,13 @@ mod tests { if i < $item_limit { assert_eq!( - pool.insert(&a), + pool.insert(a.as_reference()), Ok(InsertOutcome::NewItemInserted { committee_index: 0 }), "should accept item below limit" ); } else { assert_eq!( - pool.insert(&a), + pool.insert(a.as_reference()), Err(Error::ReachedMaxItemsPerSlot($item_limit)), "should not accept item above limit" ); diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 613411d8a2d..ff439742d11 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1358,7 +1358,7 @@ where committee_attestations.iter().skip(1).fold( attestation.clone(), |mut agg, (att, _)| { - agg.aggregate(att); + agg.aggregate(att.to_ref()); agg }, ) diff --git a/beacon_node/beacon_chain/src/validator_monitor.rs b/beacon_node/beacon_chain/src/validator_monitor.rs index ea0fe46caeb..058da369bcb 100644 --- a/beacon_node/beacon_chain/src/validator_monitor.rs +++ b/beacon_node/beacon_chain/src/validator_monitor.rs @@ -1331,7 +1331,7 @@ impl ValidatorMonitor { slot_clock, ); - let aggregator_index = signed_aggregate_and_proof.message.aggregator_index; + let aggregator_index = signed_aggregate_and_proof.message().aggregator_index(); if let Some(validator) = self.get_validator(aggregator_index) { let id = &validator.id; diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index d441596edda..11329259e90 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -2182,7 +2182,7 @@ fn verify_builder_bid( .ok() .cloned() .map(|withdrawals| Withdrawals::::from(withdrawals).tree_hash_root()); - let payload_withdrawals_root = header.withdrawals_root().ok().copied(); + let payload_withdrawals_root = header.withdrawals_root().ok(); if header.parent_hash() != parent_hash { Err(Box::new(InvalidBuilderPayload::ParentHash { diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 5580f820fbb..6cb8f6fe0b9 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3368,9 +3368,9 @@ pub fn serve( "Failure verifying aggregate and proofs"; "error" => format!("{:?}", e), "request_index" => index, - "aggregator_index" => aggregate.message.aggregator_index, - "attestation_index" => aggregate.message.aggregate.data().index, - "attestation_slot" => aggregate.message.aggregate.data().slot, + "aggregator_index" => aggregate.message().aggregator_index(), + "attestation_index" => aggregate.message().aggregate().data().index, + "attestation_slot" => aggregate.message().aggregate().data().slot, ); failures.push(api_types::Failure::new(index, format!("Verification: {:?}", e))); } @@ -3389,7 +3389,7 @@ pub fn serve( "Failure applying verified aggregate attestation to fork choice"; "error" => format!("{:?}", e), "request_index" => index, - "aggregator_index" => verified_aggregate.aggregate().message.aggregator_index, + "aggregator_index" => verified_aggregate.aggregate().message().aggregator_index(), "attestation_index" => verified_aggregate.attestation().data().index, "attestation_slot" => verified_aggregate.attestation().data().slot, ); diff --git a/beacon_node/http_api/src/publish_attestations.rs b/beacon_node/http_api/src/publish_attestations.rs index 8eaee093c1a..541ba8b7871 100644 --- a/beacon_node/http_api/src/publish_attestations.rs +++ b/beacon_node/http_api/src/publish_attestations.rs @@ -87,7 +87,7 @@ fn verify_and_publish_attestation( .send(NetworkMessage::Publish { messages: vec![PubsubMessage::Attestation(Box::new(( attestation.subnet_id(), - attestation.attestation().clone(), + attestation.attestation().clone_as_attestation(), )))], }) .map_err(|_| Error::Publication)?; diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 790cc5b70b4..3653929bdab 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -3327,8 +3327,14 @@ impl ApiTester { pub async fn test_get_validator_aggregate_and_proofs_invalid(mut self) -> Self { let mut aggregate = self.get_aggregate().await; - - aggregate.message.aggregate.data_mut().slot += 1; + match &mut aggregate { + SignedAggregateAndProof::Base(ref mut aggregate) => { + aggregate.message.aggregate.data.slot += 1; + } + SignedAggregateAndProof::Electra(ref mut aggregate) => { + aggregate.message.aggregate.data.slot += 1; + } + } self.client .post_validator_aggregate_and_proof::(&[aggregate]) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 4af0f54f31e..8945c23a549 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -9,7 +9,8 @@ use std::sync::Arc; use types::{ Attestation, AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, EthSpec, ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, - ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, + ProposerSlashing, SignedAggregateAndProof, SignedAggregateAndProofBase, + SignedAggregateAndProofElectra, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, @@ -154,10 +155,29 @@ impl PubsubMessage { // the ssz decoders match gossip_topic.kind() { GossipKind::BeaconAggregateAndProof => { - let agg_and_proof = SignedAggregateAndProof::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?; + let signed_aggregate_and_proof = + match fork_context.from_context_bytes(gossip_topic.fork_digest) { + Some(ForkName::Base) + | Some(ForkName::Altair) + | Some(ForkName::Bellatrix) + | Some(ForkName::Capella) + | Some(ForkName::Deneb) => SignedAggregateAndProof::Base( + SignedAggregateAndProofBase::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ), + Some(ForkName::Electra) => SignedAggregateAndProof::Electra( + SignedAggregateAndProofElectra::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ), + None => { + return Err(format!( + "Unknown gossipsub fork digest: {:?}", + gossip_topic.fork_digest + )) + } + }; Ok(PubsubMessage::AggregateAndProofAttestation(Box::new( - agg_and_proof, + signed_aggregate_and_proof, ))) } GossipKind::Attestation(subnet_id) => { @@ -362,9 +382,9 @@ impl std::fmt::Display for PubsubMessage { PubsubMessage::AggregateAndProofAttestation(att) => write!( f, "Aggregate and Proof: slot: {}, index: {}, aggregator_index: {}", - att.message.aggregate.data().slot, - att.message.aggregate.data().index, - att.message.aggregator_index, + att.message().aggregate().data().slot, + att.message().aggregate().data().index, + att.message().aggregator_index(), ), PubsubMessage::Attestation(data) => write!( f, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 6dc671243d5..d41233c903b 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -31,8 +31,8 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tokio::sync::mpsc; use types::{ - Attestation, AttesterSlashing, BlobSidecar, EthSpec, Hash256, IndexedAttestation, - LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, + Attestation, AttestationRef, AttesterSlashing, BlobSidecar, EthSpec, Hash256, + IndexedAttestation, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, @@ -62,8 +62,8 @@ struct VerifiedUnaggregate { /// This implementation allows `Self` to be imported to fork choice and other functions on the /// `BeaconChain`. impl VerifiedAttestation for VerifiedUnaggregate { - fn attestation(&self) -> &Attestation { - &self.attestation + fn attestation(&self) -> AttestationRef { + self.attestation.to_ref() } fn indexed_attestation(&self) -> &IndexedAttestation { @@ -95,8 +95,8 @@ struct VerifiedAggregate { /// This implementation allows `Self` to be imported to fork choice and other functions on the /// `BeaconChain`. impl VerifiedAttestation for VerifiedAggregate { - fn attestation(&self) -> &Attestation { - &self.signed_aggregate.message.aggregate + fn attestation(&self) -> AttestationRef { + self.signed_aggregate.message().aggregate() } fn indexed_attestation(&self) -> &IndexedAttestation { @@ -105,7 +105,12 @@ impl VerifiedAttestation for VerifiedAggregate { /// Efficient clone-free implementation that moves out of the `Box`. fn into_attestation_and_indices(self) -> (Attestation, Vec) { - let attestation = self.signed_aggregate.message.aggregate; + // TODO(electra): technically we shouldn't have to clone.. + let attestation = self + .signed_aggregate + .message() + .aggregate() + .clone_as_attestation(); let attesting_indices = self.indexed_attestation.attesting_indices_to_vec(); (attestation, attesting_indices) } @@ -143,10 +148,10 @@ impl FailedAtt { } } - pub fn attestation(&self) -> &Attestation { + pub fn attestation(&self) -> AttestationRef { match self { - FailedAtt::Unaggregate { attestation, .. } => attestation, - FailedAtt::Aggregate { attestation, .. } => &attestation.message.aggregate, + FailedAtt::Unaggregate { attestation, .. } => attestation.to_ref(), + FailedAtt::Aggregate { attestation, .. } => attestation.message().aggregate(), } } } @@ -412,7 +417,7 @@ impl NetworkBeaconProcessor { reprocess_tx: Option>, seen_timestamp: Duration, ) { - let beacon_block_root = aggregate.message.aggregate.data().beacon_block_root; + let beacon_block_root = aggregate.message().aggregate().data().beacon_block_root; let result = match self .chain @@ -2727,7 +2732,7 @@ impl NetworkBeaconProcessor { /// timely), propagate it on gossip. Otherwise, ignore it. fn propagate_attestation_if_timely( &self, - attestation: &Attestation, + attestation: AttestationRef, message_id: MessageId, peer_id: PeerId, ) { diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index b85be1b0c15..cc36e5cc905 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -144,7 +144,7 @@ impl NetworkBeaconProcessor { processor.process_gossip_aggregate_batch(aggregates, Some(reprocess_tx)) }; - let beacon_block_root = aggregate.message.aggregate.data().beacon_block_root; + let beacon_block_root = aggregate.message().aggregate().data().beacon_block_root; self.try_send(BeaconWorkEvent { drop_during_sync: true, work: Work::GossipAggregate { diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index f700eecf6bd..c2dc2732020 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -894,7 +894,7 @@ mod release_tests { ); for (atts, aggregate) in &attestations { - let att2 = aggregate.as_ref().unwrap().message.aggregate.clone(); + let att2 = aggregate.as_ref().unwrap().message().aggregate().clone(); let att1 = atts .into_iter() @@ -902,7 +902,7 @@ mod release_tests { .take(2) .fold::>, _>(None, |att, new_att| { if let Some(mut a) = att { - a.aggregate(&new_att); + a.aggregate(new_att.to_ref()); Some(a) } else { Some(new_att.clone()) @@ -911,9 +911,9 @@ mod release_tests { .unwrap(); let att1_indices = get_attesting_indices_from_state(&state, att1.to_ref()).unwrap(); - let att2_indices = get_attesting_indices_from_state(&state, att2.to_ref()).unwrap(); + let att2_indices = get_attesting_indices_from_state(&state, att2).unwrap(); let att1_split = SplitAttestation::new(att1.clone(), att1_indices); - let att2_split = SplitAttestation::new(att2.clone(), att2_indices); + let att2_split = SplitAttestation::new(att2.clone_as_attestation(), att2_indices); assert_eq!( att1.num_set_aggregation_bits(), @@ -1054,12 +1054,13 @@ mod release_tests { ); for (_, aggregate) in attestations { - let att = aggregate.unwrap().message.aggregate; - let attesting_indices = get_attesting_indices_from_state(&state, att.to_ref()).unwrap(); + let agg = aggregate.unwrap(); + let att = agg.message().aggregate(); + let attesting_indices = get_attesting_indices_from_state(&state, att).unwrap(); op_pool - .insert_attestation(att.clone(), attesting_indices.clone()) + .insert_attestation(att.clone_as_attestation(), attesting_indices.clone()) .unwrap(); - op_pool.insert_attestation(att, attesting_indices).unwrap(); + op_pool.insert_attestation(att.clone_as_attestation(), attesting_indices).unwrap(); } assert_eq!(op_pool.num_attestations(), committees.len()); @@ -1108,7 +1109,7 @@ mod release_tests { None, |att, new_att| { if let Some(mut a) = att { - a.aggregate(new_att); + a.aggregate(new_att.to_ref()); Some(a) } else { Some(new_att.clone()) @@ -1131,7 +1132,7 @@ mod release_tests { None, |att, new_att| { if let Some(mut a) = att { - a.aggregate(new_att); + a.aggregate(new_att.to_ref()); Some(a) } else { Some(new_att.clone()) @@ -1208,7 +1209,7 @@ mod release_tests { .fold::, _>( att_0.clone(), |mut att, new_att| { - att.aggregate(new_att); + att.aggregate(new_att.to_ref()); att }, ) @@ -1304,7 +1305,7 @@ mod release_tests { .fold::, _>( att_0.clone(), |mut att, new_att| { - att.aggregate(new_att); + att.aggregate(new_att.to_ref()); att }, ) diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index 9885daaab0a..a179583556f 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -425,7 +425,7 @@ where E: EthSpec, F: Fn(usize) -> Option>, { - let slot = signed_aggregate_and_proof.message.aggregate.data().slot; + let slot = signed_aggregate_and_proof.message().aggregate().data().slot; let domain = spec.get_domain( slot.epoch(E::slots_per_epoch()), @@ -434,8 +434,8 @@ where genesis_validators_root, ); let message = slot.signing_root(domain); - let signature = &signed_aggregate_and_proof.message.selection_proof; - let validator_index = signed_aggregate_and_proof.message.aggregator_index; + let signature = signed_aggregate_and_proof.message().selection_proof(); + let validator_index = signed_aggregate_and_proof.message().aggregator_index(); Ok(SignatureSet::single_pubkey( signature, @@ -456,8 +456,8 @@ where F: Fn(usize) -> Option>, { let target_epoch = signed_aggregate_and_proof - .message - .aggregate + .message() + .aggregate() .data() .target .epoch; @@ -468,9 +468,9 @@ where fork, genesis_validators_root, ); - let message = signed_aggregate_and_proof.message.signing_root(domain); - let signature = &signed_aggregate_and_proof.signature; - let validator_index = signed_aggregate_and_proof.message.aggregator_index; + let message = signed_aggregate_and_proof.message().signing_root(domain); + let signature = signed_aggregate_and_proof.signature(); + let validator_index = signed_aggregate_and_proof.message().aggregator_index(); Ok(SignatureSet::single_pubkey( signature, diff --git a/consensus/types/src/aggregate_and_proof.rs b/consensus/types/src/aggregate_and_proof.rs index 0297c6d6845..b0dbdadb952 100644 --- a/consensus/types/src/aggregate_and_proof.rs +++ b/consensus/types/src/aggregate_and_proof.rs @@ -1,41 +1,79 @@ +use super::{Attestation, AttestationBase, AttestationElectra, AttestationRef}; use super::{ - Attestation, ChainSpec, Domain, EthSpec, Fork, Hash256, PublicKey, SecretKey, SelectionProof, - Signature, SignedRoot, + ChainSpec, Domain, EthSpec, Fork, Hash256, PublicKey, SecretKey, SelectionProof, Signature, + SignedRoot, }; use crate::test_utils::TestRandom; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; +use superstruct::superstruct; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -/// A Validators aggregate attestation and selection proof. -/// -/// Spec v0.12.1 +#[superstruct( + variants(Base, Electra), + variant_attributes( + derive( + arbitrary::Arbitrary, + Debug, + Clone, + PartialEq, + Serialize, + Deserialize, + Encode, + Decode, + TestRandom, + TreeHash, + ), + serde(bound = "E: EthSpec"), + arbitrary(bound = "E: EthSpec"), + ), + ref_attributes( + derive(Debug, PartialEq, TreeHash, Serialize,), + serde(bound = "E: EthSpec"), + tree_hash(enum_behaviour = "transparent") + ) +)] #[derive( - arbitrary::Arbitrary, - Debug, - Clone, - PartialEq, - Serialize, - Deserialize, - Encode, - Decode, - TestRandom, - TreeHash, + arbitrary::Arbitrary, Debug, Clone, PartialEq, Serialize, Deserialize, Encode, TreeHash, )] -#[serde(bound = "E: EthSpec")] +#[serde(untagged)] +#[tree_hash(enum_behaviour = "transparent")] +#[ssz(enum_behaviour = "transparent")] +#[serde(bound = "E: EthSpec", deny_unknown_fields)] #[arbitrary(bound = "E: EthSpec")] pub struct AggregateAndProof { /// The index of the validator that created the attestation. #[serde(with = "serde_utils::quoted_u64")] + #[superstruct(getter(copy))] pub aggregator_index: u64, /// The aggregate attestation. + #[superstruct(flatten)] pub aggregate: Attestation, /// A proof provided by the validator that permits them to publish on the /// `beacon_aggregate_and_proof` gossipsub topic. pub selection_proof: Signature, } +impl<'a, E: EthSpec> AggregateAndProofRef<'a, E> { + /// Returns `true` if `validator_pubkey` signed over `self.aggregate.data.slot`. + pub fn aggregate(self) -> AttestationRef<'a, E> { + match self { + AggregateAndProofRef::Base(a) => AttestationRef::Base(&a.aggregate), + AggregateAndProofRef::Electra(a) => AttestationRef::Electra(&a.aggregate), + } + } +} +impl AggregateAndProof { + /// Returns `true` if `validator_pubkey` signed over `self.aggregate.data.slot`. + pub fn aggregate(&self) -> AttestationRef { + match self { + AggregateAndProof::Base(a) => AttestationRef::Base(&a.aggregate), + AggregateAndProof::Electra(a) => AttestationRef::Electra(&a.aggregate), + } + } +} + impl AggregateAndProof { /// Produces a new `AggregateAndProof` with a `selection_proof` generated by signing /// `aggregate.data.slot` with `secret_key`. @@ -62,10 +100,17 @@ impl AggregateAndProof { }) .into(); - Self { - aggregator_index, - aggregate, - selection_proof, + match aggregate { + Attestation::Base(attestation) => Self::Base(AggregateAndProofBase { + aggregator_index, + aggregate: attestation, + selection_proof, + }), + Attestation::Electra(attestation) => Self::Electra(AggregateAndProofElectra { + aggregator_index, + aggregate: attestation, + selection_proof, + }), } } @@ -77,16 +122,17 @@ impl AggregateAndProof { genesis_validators_root: Hash256, spec: &ChainSpec, ) -> bool { - let target_epoch = self.aggregate.data().slot.epoch(E::slots_per_epoch()); + let target_epoch = self.aggregate().data().slot.epoch(E::slots_per_epoch()); let domain = spec.get_domain( target_epoch, Domain::SelectionProof, fork, genesis_validators_root, ); - let message = self.aggregate.data().slot.signing_root(domain); - self.selection_proof.verify(validator_pubkey, message) + let message = self.aggregate().data().slot.signing_root(domain); + self.selection_proof().verify(validator_pubkey, message) } } impl SignedRoot for AggregateAndProof {} +impl<'a, E: EthSpec> SignedRoot for AggregateAndProofRef<'a, E> {} diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 0d3f29ae289..4676a100be6 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -42,7 +42,8 @@ pub enum Error { derivative(PartialEq, Hash(bound = "E: EthSpec")), serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), - ) + ), + ref_attributes(derive(TreeHash), tree_hash(enum_behaviour = "transparent")) )] #[derive( Debug, @@ -123,22 +124,24 @@ impl Attestation { /// Aggregate another Attestation into this one. /// /// The aggregation bitfields must be disjoint, and the data must be the same. - pub fn aggregate(&mut self, other: &Self) { + pub fn aggregate(&mut self, other: AttestationRef) { match self { - Attestation::Base(att) => { - debug_assert!(other.as_base().is_ok()); - - if let Ok(other) = other.as_base() { - att.aggregate(other) + Attestation::Base(att) => match other { + AttestationRef::Base(oth) => { + att.aggregate(oth); } - } - Attestation::Electra(att) => { - debug_assert!(other.as_electra().is_ok()); - - if let Ok(other) = other.as_electra() { - att.aggregate(other) + AttestationRef::Electra(_) => { + debug_assert!(false, "Cannot aggregate base and electra attestations"); + } + }, + Attestation::Electra(att) => match other { + AttestationRef::Base(_) => { + debug_assert!(false, "Cannot aggregate base and electra attestations"); } - } + AttestationRef::Electra(oth) => { + att.aggregate(oth); + } + }, } } @@ -215,8 +218,22 @@ impl Attestation { impl<'a, E: EthSpec> AttestationRef<'a, E> { pub fn clone_as_attestation(self) -> Attestation { match self { - AttestationRef::Base(att) => Attestation::Base(att.clone()), - AttestationRef::Electra(att) => Attestation::Electra(att.clone()), + Self::Base(att) => Attestation::Base(att.clone()), + Self::Electra(att) => Attestation::Electra(att.clone()), + } + } + + pub fn is_aggregation_bits_zero(self) -> bool { + match self { + Self::Base(att) => att.aggregation_bits.is_zero(), + Self::Electra(att) => att.aggregation_bits.is_zero(), + } + } + + pub fn num_set_aggregation_bits(&self) -> usize { + match self { + Self::Base(att) => att.aggregation_bits.num_set_bits(), + Self::Electra(att) => att.aggregation_bits.num_set_bits(), } } } diff --git a/consensus/types/src/attester_slashing.rs b/consensus/types/src/attester_slashing.rs index 4ad19f477ca..6668f809b82 100644 --- a/consensus/types/src/attester_slashing.rs +++ b/consensus/types/src/attester_slashing.rs @@ -39,15 +39,10 @@ use tree_hash_derive::TreeHash; #[ssz(enum_behaviour = "transparent")] #[tree_hash(enum_behaviour = "transparent")] pub struct AttesterSlashing { - // TODO(electra) change this to `#[superstruct(flatten)]` when 0.8 is out.. - #[superstruct(only(Base), partial_getter(rename = "attestation_1_base"))] - pub attestation_1: IndexedAttestationBase, - #[superstruct(only(Electra), partial_getter(rename = "attestation_1_electra"))] - pub attestation_1: IndexedAttestationElectra, - #[superstruct(only(Base), partial_getter(rename = "attestation_2_base"))] - pub attestation_2: IndexedAttestationBase, - #[superstruct(only(Electra), partial_getter(rename = "attestation_2_electra"))] - pub attestation_2: IndexedAttestationElectra, + #[superstruct(flatten)] + pub attestation_1: IndexedAttestation, + #[superstruct(flatten)] + pub attestation_2: IndexedAttestation, } /// This is a copy of the `AttesterSlashing` enum but with `Encode` and `Decode` derived diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 324d7b97472..2d7bc950714 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -77,16 +77,13 @@ pub struct ExecutionPayloadHeader { pub block_hash: ExecutionBlockHash, #[superstruct(getter(copy))] pub transactions_root: Hash256, - #[superstruct(only(Capella, Deneb, Electra))] - #[superstruct(getter(copy))] + #[superstruct(only(Capella, Deneb, Electra), partial_getter(copy))] pub withdrawals_root: Hash256, - #[superstruct(only(Deneb, Electra))] + #[superstruct(only(Deneb, Electra), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] - #[superstruct(getter(copy))] pub blob_gas_used: u64, - #[superstruct(only(Deneb, Electra))] + #[superstruct(only(Deneb, Electra), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] - #[superstruct(getter(copy))] pub excess_blob_gas: u64, #[superstruct(only(Electra), partial_getter(copy))] pub deposit_receipts_root: Hash256, diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index ebfc4ab8bb0..3a1c0b9a52c 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -114,7 +114,9 @@ pub mod runtime_var_list; use ethereum_types::{H160, H256}; pub use crate::activation_queue::ActivationQueue; -pub use crate::aggregate_and_proof::AggregateAndProof; +pub use crate::aggregate_and_proof::{ + AggregateAndProof, AggregateAndProofBase, AggregateAndProofElectra, AggregateAndProofRef, +}; pub use crate::attestation::{ Attestation, AttestationBase, AttestationElectra, AttestationRef, AttestationRefMut, Error as AttestationError, @@ -218,7 +220,9 @@ pub use crate::relative_epoch::{Error as RelativeEpochError, RelativeEpoch}; pub use crate::runtime_var_list::RuntimeVariableList; pub use crate::selection_proof::SelectionProof; pub use crate::shuffling_id::AttestationShufflingId; -pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof; +pub use crate::signed_aggregate_and_proof::{ + SignedAggregateAndProof, SignedAggregateAndProofBase, SignedAggregateAndProofElectra, +}; pub use crate::signed_beacon_block::{ ssz_tagged_signed_beacon_block, ssz_tagged_signed_beacon_block_arc, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, diff --git a/consensus/types/src/signed_aggregate_and_proof.rs b/consensus/types/src/signed_aggregate_and_proof.rs index 491edb3cb0d..649b5fc6fac 100644 --- a/consensus/types/src/signed_aggregate_and_proof.rs +++ b/consensus/types/src/signed_aggregate_and_proof.rs @@ -1,10 +1,14 @@ use super::{ - AggregateAndProof, Attestation, ChainSpec, Domain, EthSpec, Fork, Hash256, SecretKey, - SelectionProof, Signature, SignedRoot, + AggregateAndProof, AggregateAndProofBase, AggregateAndProofElectra, AggregateAndProofRef, +}; +use super::{ + Attestation, ChainSpec, Domain, EthSpec, Fork, Hash256, SecretKey, SelectionProof, Signature, + SignedRoot, }; use crate::test_utils::TestRandom; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; +use superstruct::superstruct; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; @@ -12,22 +16,36 @@ use tree_hash_derive::TreeHash; /// gossipsub topic. /// /// Spec v0.12.1 +#[superstruct( + variants(Base, Electra), + variant_attributes( + derive( + arbitrary::Arbitrary, + Debug, + Clone, + PartialEq, + Serialize, + Deserialize, + Encode, + Decode, + TestRandom, + TreeHash, + ), + serde(bound = "E: EthSpec"), + arbitrary(bound = "E: EthSpec"), + ) +)] #[derive( - Debug, - Clone, - PartialEq, - Serialize, - Deserialize, - Encode, - Decode, - TestRandom, - TreeHash, - arbitrary::Arbitrary, + arbitrary::Arbitrary, Debug, Clone, PartialEq, Serialize, Deserialize, Encode, TreeHash, )] -#[serde(bound = "E: EthSpec")] +#[serde(untagged)] +#[tree_hash(enum_behaviour = "transparent")] +#[ssz(enum_behaviour = "transparent")] +#[serde(bound = "E: EthSpec", deny_unknown_fields)] #[arbitrary(bound = "E: EthSpec")] pub struct SignedAggregateAndProof { /// The `AggregateAndProof` that was signed. + #[superstruct(flatten)] pub message: AggregateAndProof, /// The aggregate attestation. pub signature: Signature, @@ -57,7 +75,7 @@ impl SignedAggregateAndProof { spec, ); - let target_epoch = message.aggregate.data().slot.epoch(E::slots_per_epoch()); + let target_epoch = message.aggregate().data().slot.epoch(E::slots_per_epoch()); let domain = spec.get_domain( target_epoch, Domain::AggregateAndProof, @@ -66,9 +84,28 @@ impl SignedAggregateAndProof { ); let signing_message = message.signing_root(domain); - SignedAggregateAndProof { - message, - signature: secret_key.sign(signing_message), + match message { + AggregateAndProof::Base(message) => { + SignedAggregateAndProof::Base(SignedAggregateAndProofBase { + message, + signature: secret_key.sign(signing_message), + }) + } + AggregateAndProof::Electra(message) => { + SignedAggregateAndProof::Electra(SignedAggregateAndProofElectra { + message, + signature: secret_key.sign(signing_message), + }) + } + } + } + + pub fn message(&self) -> AggregateAndProofRef { + match self { + SignedAggregateAndProof::Base(message) => AggregateAndProofRef::Base(&message.message), + SignedAggregateAndProof::Electra(message) => { + AggregateAndProofRef::Electra(&message.message) + } } } } diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 7b60e331666..676bf7cae2e 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -24,8 +24,8 @@ use std::future::Future; use std::sync::Arc; use std::time::Duration; use types::{ - Attestation, AttesterSlashing, AttesterSlashingRef, BeaconBlock, BeaconState, BlobSidecar, - BlobsList, Checkpoint, ExecutionBlockHash, Hash256, IndexedAttestation, KzgProof, + Attestation, AttestationRef, AttesterSlashing, AttesterSlashingRef, BeaconBlock, BeaconState, + BlobSidecar, BlobsList, Checkpoint, ExecutionBlockHash, Hash256, IndexedAttestation, KzgProof, ProposerPreparationData, SignedBeaconBlock, Slot, Uint256, }; @@ -589,11 +589,11 @@ impl Tester { } pub fn process_attestation(&self, attestation: &Attestation) -> Result<(), Error> { - let (indexed_attestation, _) = - obtain_indexed_attestation_and_committees_per_slot(&self.harness.chain, attestation) - .map_err(|e| { - Error::InternalError(format!("attestation indexing failed with {:?}", e)) - })?; + let (indexed_attestation, _) = obtain_indexed_attestation_and_committees_per_slot( + &self.harness.chain, + attestation.to_ref(), + ) + .map_err(|e| Error::InternalError(format!("attestation indexing failed with {:?}", e)))?; let verified_attestation: ManuallyVerifiedAttestation> = ManuallyVerifiedAttestation { attestation, @@ -865,8 +865,8 @@ pub struct ManuallyVerifiedAttestation<'a, T: BeaconChainTypes> { } impl<'a, T: BeaconChainTypes> VerifiedAttestation for ManuallyVerifiedAttestation<'a, T> { - fn attestation(&self) -> &Attestation { - self.attestation + fn attestation(&self) -> AttestationRef { + self.attestation.to_ref() } fn indexed_attestation(&self) -> &IndexedAttestation { diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 653d829d685..5b7f31867bb 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -605,11 +605,11 @@ impl AttestationService { { Ok(()) => { for signed_aggregate_and_proof in signed_aggregate_and_proofs { - let attestation = &signed_aggregate_and_proof.message.aggregate; + let attestation = signed_aggregate_and_proof.message().aggregate(); info!( log, "Successfully published attestation"; - "aggregator" => signed_aggregate_and_proof.message.aggregator_index, + "aggregator" => signed_aggregate_and_proof.message().aggregator_index(), "signatures" => attestation.num_set_aggregation_bits(), "head_block" => format!("{:?}", attestation.data().beacon_block_root), "committee_index" => attestation.data().index, @@ -620,12 +620,12 @@ impl AttestationService { } Err(e) => { for signed_aggregate_and_proof in signed_aggregate_and_proofs { - let attestation = &signed_aggregate_and_proof.message.aggregate; + let attestation = &signed_aggregate_and_proof.message().aggregate(); crit!( log, "Failed to publish attestation"; "error" => %e, - "aggregator" => signed_aggregate_and_proof.message.aggregator_index, + "aggregator" => signed_aggregate_and_proof.message().aggregator_index(), "committee_index" => attestation.data().index, "slot" => attestation.data().slot.as_u64(), "type" => "aggregated", diff --git a/validator_client/src/signing_method.rs b/validator_client/src/signing_method.rs index fe520e11f5f..19824d76fb0 100644 --- a/validator_client/src/signing_method.rs +++ b/validator_client/src/signing_method.rs @@ -38,7 +38,7 @@ pub enum SignableMessage<'a, E: EthSpec, Payload: AbstractExecPayload = FullP RandaoReveal(Epoch), BeaconBlock(&'a BeaconBlock), AttestationData(&'a AttestationData), - SignedAggregateAndProof(&'a AggregateAndProof), + SignedAggregateAndProof(AggregateAndProofRef<'a, E>), SelectionProof(Slot), SyncSelectionProof(&'a SyncAggregatorSelectionData), SyncCommitteeSignature { diff --git a/validator_client/src/signing_method/web3signer.rs b/validator_client/src/signing_method/web3signer.rs index 8ad37a1620a..86e7015ad35 100644 --- a/validator_client/src/signing_method/web3signer.rs +++ b/validator_client/src/signing_method/web3signer.rs @@ -43,7 +43,7 @@ pub enum Web3SignerObject<'a, E: EthSpec, Payload: AbstractExecPayload> { AggregationSlot { slot: Slot, }, - AggregateAndProof(&'a AggregateAndProof), + AggregateAndProof(AggregateAndProofRef<'a, E>), Attestation(&'a AttestationData), BeaconBlock { version: ForkName, diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 8e69cb440a4..a173e5da12b 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -18,12 +18,16 @@ use std::sync::Arc; use task_executor::TaskExecutor; use types::{ attestation::Error as AttestationError, graffiti::GraffitiString, AbstractExecPayload, Address, - AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof, - Domain, Epoch, EthSpec, Fork, ForkName, Graffiti, Hash256, PublicKeyBytes, SelectionProof, - Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedRoot, - SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, - SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, - ValidatorRegistrationData, VoluntaryExit, + Attestation, BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof, Domain, Epoch, + EthSpec, Fork, ForkName, Graffiti, Hash256, PublicKeyBytes, SelectionProof, Signature, + SignedBeaconBlock, SignedContributionAndProof, SignedRoot, SignedValidatorRegistrationData, + SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, + SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, + VoluntaryExit, +}; +use types::{ + AggregateAndProof, AggregateAndProofBase, AggregateAndProofElectra, SignedAggregateAndProof, + SignedAggregateAndProofBase, SignedAggregateAndProofElectra, }; pub use crate::doppelganger_service::DoppelgangerStatus; @@ -801,16 +805,23 @@ impl ValidatorStore { let signing_epoch = aggregate.data().target.epoch; let signing_context = self.signing_context(Domain::AggregateAndProof, signing_epoch); - let message = AggregateAndProof { - aggregator_index, - aggregate, - selection_proof: selection_proof.into(), + let message = match aggregate { + Attestation::Base(att) => AggregateAndProof::Base(AggregateAndProofBase { + aggregator_index, + aggregate: att, + selection_proof: selection_proof.into(), + }), + Attestation::Electra(att) => AggregateAndProof::Electra(AggregateAndProofElectra { + aggregator_index, + aggregate: att, + selection_proof: selection_proof.into(), + }), }; let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method .get_signature::>( - SignableMessage::SignedAggregateAndProof(&message), + SignableMessage::SignedAggregateAndProof(message.to_ref()), signing_context, &self.spec, &self.task_executor, @@ -819,7 +830,17 @@ impl ValidatorStore { metrics::inc_counter_vec(&metrics::SIGNED_AGGREGATES_TOTAL, &[metrics::SUCCESS]); - Ok(SignedAggregateAndProof { message, signature }) + match message { + AggregateAndProof::Base(message) => { + Ok(SignedAggregateAndProof::Base(SignedAggregateAndProofBase { + message, + signature, + })) + } + AggregateAndProof::Electra(message) => Ok(SignedAggregateAndProof::Electra( + SignedAggregateAndProofElectra { message, signature }, + )), + } } /// Produces a `SelectionProof` for the `slot`, signed by with corresponding secret key to From 38382a3ca1fd0d65300160dbe6608e2d267df110 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 6 May 2024 17:32:25 -0400 Subject: [PATCH 006/227] cargo fmt --- beacon_node/operation_pool/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index c2dc2732020..6645416d4b0 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -1060,7 +1060,9 @@ mod release_tests { op_pool .insert_attestation(att.clone_as_attestation(), attesting_indices.clone()) .unwrap(); - op_pool.insert_attestation(att.clone_as_attestation(), attesting_indices).unwrap(); + op_pool + .insert_attestation(att.clone_as_attestation(), attesting_indices) + .unwrap(); } assert_eq!(op_pool.num_attestations(), committees.len()); From e2e82ff1b99bef656224564d3a45266093a46266 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 6 May 2024 18:56:16 -0400 Subject: [PATCH 007/227] process withdrawals updates --- beacon_node/beacon_chain/src/beacon_chain.rs | 5 +- .../beacon_chain/src/execution_payload.rs | 2 +- beacon_node/http_api/src/builder_states.rs | 2 +- .../http_api/tests/interactive_tests.rs | 2 + beacon_node/http_api/tests/tests.rs | 4 +- .../src/per_block_processing.rs | 76 +++++++++++++++++-- consensus/types/src/beacon_state.rs | 4 +- consensus/types/src/validator.rs | 21 +++-- 8 files changed, 99 insertions(+), 17 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index cd47068977b..99d8eac8aac 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4435,6 +4435,7 @@ impl BeaconChain { let proposal_epoch = proposal_slot.epoch(T::EthSpec::slots_per_epoch()); if head_state.current_epoch() == proposal_epoch { return get_expected_withdrawals(&unadvanced_state, &self.spec) + .map(|(withdrawals, _)| withdrawals) .map_err(Error::PrepareProposerFailed); } @@ -4452,7 +4453,9 @@ impl BeaconChain { proposal_epoch.start_slot(T::EthSpec::slots_per_epoch()), &self.spec, )?; - get_expected_withdrawals(&advanced_state, &self.spec).map_err(Error::PrepareProposerFailed) + get_expected_withdrawals(&advanced_state, &self.spec) + .map(|(withdrawals, _)| withdrawals) + .map_err(Error::PrepareProposerFailed) } /// Determine whether a fork choice update to the execution layer should be overridden. diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index cbffe363422..a6e0d247dc2 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -413,7 +413,7 @@ pub fn get_execution_payload( state.latest_execution_payload_header()?.block_hash(); let withdrawals = match state { &BeaconState::Capella(_) | &BeaconState::Deneb(_) | &BeaconState::Electra(_) => { - Some(get_expected_withdrawals(state, spec)?.into()) + Some(get_expected_withdrawals(state, spec)?.0.into()) } &BeaconState::Bellatrix(_) => None, // These shouldn't happen but they're here to make the pattern irrefutable diff --git a/beacon_node/http_api/src/builder_states.rs b/beacon_node/http_api/src/builder_states.rs index a540113ab43..54f2c0efa8d 100644 --- a/beacon_node/http_api/src/builder_states.rs +++ b/beacon_node/http_api/src/builder_states.rs @@ -33,7 +33,7 @@ pub fn get_next_withdrawals( } match get_expected_withdrawals(&state, &chain.spec) { - Ok(withdrawals) => Ok(withdrawals), + Ok((withdrawals, _)) => Ok(withdrawals), Err(e) => Err(warp_utils::reject::custom_server_error(format!( "failed to get expected withdrawal: {:?}", e diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 529dc852e98..711820ccac6 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -610,6 +610,7 @@ pub async fn proposer_boost_re_org_test( assert_eq!(state_b.slot(), slot_b); let pre_advance_withdrawals = get_expected_withdrawals(&state_b, &harness.chain.spec) .unwrap() + .0 .to_vec(); complete_state_advance(&mut state_b, None, slot_c, &harness.chain.spec).unwrap(); @@ -696,6 +697,7 @@ pub async fn proposer_boost_re_org_test( get_expected_withdrawals(&state_b, &harness.chain.spec) } .unwrap() + .0 .to_vec(); let payload_attribs_withdrawals = payload_attribs.withdrawals().unwrap(); assert_eq!(expected_withdrawals, *payload_attribs_withdrawals); diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 3653929bdab..0c246dc7d94 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -5448,7 +5448,9 @@ impl ApiTester { &self.chain.spec, ); } - let expected_withdrawals = get_expected_withdrawals(&state, &self.chain.spec).unwrap(); + let expected_withdrawals = get_expected_withdrawals(&state, &self.chain.spec) + .unwrap() + .0; // fetch expected withdrawals from the client let result = self.client.get_expected_withdrawals(&state_id).await; diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 98671f82b9f..4be4ac9581f 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -503,13 +503,61 @@ pub fn compute_timestamp_at_slot( pub fn get_expected_withdrawals( state: &BeaconState, spec: &ChainSpec, -) -> Result, BlockProcessingError> { +) -> Result<(Withdrawals, Option), BlockProcessingError> { let epoch = state.current_epoch(); let mut withdrawal_index = state.next_withdrawal_index()?; let mut validator_index = state.next_withdrawal_validator_index()?; let mut withdrawals = vec![]; let fork_name = state.fork_name_unchecked(); + // [New in Electra:EIP7251] + // Consume pending partial withdrawals + let partial_withdrawals_count = + if let Ok(partial_withdrawals) = state.pending_partial_withdrawals() { + for withdrawal in partial_withdrawals { + if withdrawal.withdrawable_epoch > epoch + || withdrawals.len() == spec.max_pending_partials_per_withdrawals_sweep as usize + { + break; + } + + let withdrawal_balance = state + .balances() + .get(withdrawal.index as usize) + .copied() + .ok_or(BeaconStateError::BalancesOutOfBounds( + withdrawal.index as usize, + ))?; + let validator = state.validators().get(withdrawal.index as usize).ok_or( + BeaconStateError::UnknownValidator(withdrawal.index as usize), + )?; + + let has_sufficient_effective_balance = + validator.effective_balance >= spec.min_activation_balance; + let has_excess_balance = withdrawal_balance > spec.min_activation_balance; + + if validator.exit_epoch == spec.far_future_epoch + && has_sufficient_effective_balance + && has_excess_balance + { + let withdrawable_balance = std::cmp::min( + withdrawal_balance.safe_sub(spec.min_activation_balance)?, + withdrawal.amount, + ); + withdrawals.push(Withdrawal { + index: withdrawal_index, + validator_index: withdrawal.index, + address: Address::from_slice(&validator.withdrawal_credentials[12..]), + amount: withdrawable_balance, + }); + withdrawal_index.safe_add_assign(1)?; + } + } + Some(withdrawals.len()) + } else { + None + }; + let bound = std::cmp::min( state.validators().len() as u64, spec.max_validators_per_withdrawals_sweep, @@ -536,7 +584,10 @@ pub fn get_expected_withdrawals( address: validator .get_eth1_withdrawal_address(spec) .ok_or(BlockProcessingError::WithdrawalCredentialsInvalid)?, - amount: balance.safe_sub(spec.max_effective_balance)?, + amount: balance.safe_sub( + validator + .get_validator_max_effective_balance(spec, state.fork_name_unchecked()), + )?, }); withdrawal_index.safe_add_assign(1)?; } @@ -548,7 +599,7 @@ pub fn get_expected_withdrawals( .safe_rem(state.validators().len() as u64)?; } - Ok(withdrawals.into()) + Ok((withdrawals.into(), partial_withdrawals_count)) } /// Apply withdrawals to the state. @@ -558,9 +609,9 @@ pub fn process_withdrawals>( spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { match state { - BeaconState::Bellatrix(_) => Ok(()), BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => { - let expected_withdrawals = get_expected_withdrawals(state, spec)?; + let (expected_withdrawals, partial_withdrawals_count) = + get_expected_withdrawals(state, spec)?; let expected_root = expected_withdrawals.tree_hash_root(); let withdrawals_root = payload.withdrawals_root()?; @@ -579,6 +630,19 @@ pub fn process_withdrawals>( )?; } + // Update pending partial withdrawals [New in Electra:EIP7251] + if let Some(partial_withdrawals_count) = partial_withdrawals_count { + let new_partial_withdrawals = state + .pending_partial_withdrawals()? + .to_vec() + .get(partial_withdrawals_count..) + .ok_or(BeaconStateError::PartialWithdrawalCountInvalid( + partial_withdrawals_count, + ))? + .to_vec(); + *state.pending_partial_withdrawals_mut()? = List::new(new_partial_withdrawals)?; + } + // Update the next withdrawal index if this block contained withdrawals if let Some(latest_withdrawal) = expected_withdrawals.last() { *state.next_withdrawal_index_mut()? = latest_withdrawal.index.safe_add(1)?; @@ -606,6 +670,6 @@ pub fn process_withdrawals>( Ok(()) } // these shouldn't even be encountered but they're here for completeness - BeaconState::Base(_) | BeaconState::Altair(_) => Ok(()), + BeaconState::Base(_) | BeaconState::Altair(_) | BeaconState::Bellatrix(_) => Ok(()), } } diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 577f282a556..dda87c5dd50 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -159,6 +159,7 @@ pub enum Error { IndexNotSupported(usize), InvalidFlagIndex(usize), MerkleTreeError(merkle_proof::MerkleTreeError), + PartialWithdrawalCountInvalid(usize), } /// Control whether an epoch-indexed field can be indexed at the next epoch or not. @@ -2097,11 +2098,12 @@ impl BeaconState { &self, validator_index: usize, spec: &ChainSpec, + current_fork: ForkName, ) -> Result { let max_effective_balance = self .validators() .get(validator_index) - .map(|validator| validator.get_validator_max_effective_balance(spec)) + .map(|validator| validator.get_validator_max_effective_balance(spec, current_fork)) .ok_or(Error::UnknownValidator(validator_index))?; Ok(std::cmp::min( *self diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index 9e26d1eeca6..3287942dfe7 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -203,7 +203,7 @@ impl Validator { current_fork: ForkName, ) -> bool { if current_fork >= ForkName::Electra { - self.is_partially_withdrawable_validator_electra(balance, spec) + self.is_partially_withdrawable_validator_electra(balance, spec, current_fork) } else { self.is_partially_withdrawable_validator_capella(balance, spec) } @@ -223,8 +223,9 @@ impl Validator { &self, balance: u64, spec: &ChainSpec, + current_fork: ForkName, ) -> bool { - let max_effective_balance = self.get_validator_max_effective_balance(spec); + let max_effective_balance = self.get_validator_max_effective_balance(spec, current_fork); let has_max_effective_balance = self.effective_balance == max_effective_balance; let has_excess_balance = balance > max_effective_balance; self.has_execution_withdrawal_credential(spec) @@ -239,11 +240,19 @@ impl Validator { } /// Returns the max effective balance for a validator in gwei. - pub fn get_validator_max_effective_balance(&self, spec: &ChainSpec) -> u64 { - if self.has_compounding_withdrawal_credential(spec) { - spec.max_effective_balance_electra + pub fn get_validator_max_effective_balance( + &self, + spec: &ChainSpec, + current_fork: ForkName, + ) -> u64 { + if current_fork >= ForkName::Electra { + if self.has_compounding_withdrawal_credential(spec) { + spec.max_effective_balance_electra + } else { + spec.min_activation_balance + } } else { - spec.min_activation_balance + spec.max_effective_balance } } } From e0abede1d174c386e234911feddc2b0b122a0932 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 6 May 2024 21:25:47 -0400 Subject: [PATCH 008/227] cleanup withdrawals processing --- .../src/per_block_processing.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 4be4ac9581f..b70579c8263 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -547,7 +547,13 @@ pub fn get_expected_withdrawals( withdrawals.push(Withdrawal { index: withdrawal_index, validator_index: withdrawal.index, - address: Address::from_slice(&validator.withdrawal_credentials[12..]), + address: Address::from_slice( + validator + .withdrawal_credentials + .0 + .get(12..) + .ok_or(BeaconStateError::IndexNotSupported(12))?, + ), amount: withdrawable_balance, }); withdrawal_index.safe_add_assign(1)?; @@ -634,12 +640,9 @@ pub fn process_withdrawals>( if let Some(partial_withdrawals_count) = partial_withdrawals_count { let new_partial_withdrawals = state .pending_partial_withdrawals()? - .to_vec() - .get(partial_withdrawals_count..) - .ok_or(BeaconStateError::PartialWithdrawalCountInvalid( - partial_withdrawals_count, - ))? - .to_vec(); + .iter_from(partial_withdrawals_count)? + .cloned() + .collect::>(); *state.pending_partial_withdrawals_mut()? = List::new(new_partial_withdrawals)?; } From f1f9f92dec41d890d665cc51ed60b93e4eef0774 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 6 May 2024 21:56:50 -0400 Subject: [PATCH 009/227] =?UTF-8?q?update=20`process=5Foperations`=C2=A0de?= =?UTF-8?q?posit=20length=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../process_operations.rs | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 9b4e9d71394..9aaef8f965a 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -357,17 +357,34 @@ pub fn process_deposits( deposits: &[Deposit], spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { - let expected_deposit_len = std::cmp::min( - E::MaxDeposits::to_u64(), - state.get_outstanding_deposit_len()?, - ); - block_verify!( - deposits.len() as u64 == expected_deposit_len, - BlockProcessingError::DepositCountInvalid { - expected: expected_deposit_len as usize, - found: deposits.len(), - } - ); + // [Modified in Electra:EIP6110] + // Disable former deposit mechanism once all prior deposits are processed + // + // If `deposit_receipts_start_index` does not exist as a field on `state`, electra is disabled + // which means we always want to use the old check, so this field defaults to `u64::MAX`. + let eth1_deposit_index_limit = state.deposit_receipts_start_index().unwrap_or(u64::MAX); + + if state.eth1_deposit_index() < eth1_deposit_index_limit { + let expected_deposit_len = std::cmp::min( + E::MaxDeposits::to_u64(), + state.get_outstanding_deposit_len()?, + ); + block_verify!( + deposits.len() as u64 == expected_deposit_len, + BlockProcessingError::DepositCountInvalid { + expected: expected_deposit_len as usize, + found: deposits.len(), + } + ); + } else { + block_verify!( + deposits.len() as u64 == 0, + BlockProcessingError::DepositCountInvalid { + expected: 0, + found: deposits.len(), + } + ); + } // Verify merkle proofs in parallel. deposits From 3c688410ccc9e981c3f732c93473c2ffd73b7a9a Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 7 May 2024 08:48:01 -0400 Subject: [PATCH 010/227] add apply_deposit changes --- .../src/per_block_processing.rs | 2 +- .../process_operations.rs | 54 ++++++++++++++++--- .../per_block_processing/verify_deposit.rs | 2 +- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index b70579c8263..43414f70859 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -20,7 +20,7 @@ pub use verify_attestation::{ }; pub use verify_bls_to_execution_change::verify_bls_to_execution_change; pub use verify_deposit::{ - get_existing_validator_index, verify_deposit_merkle_proof, verify_deposit_signature, + get_existing_validator_index, is_valid_deposit_signature, verify_deposit_merkle_proof, }; pub use verify_exit::verify_exit; diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 9aaef8f965a..35a0c0ffadb 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -6,6 +6,7 @@ use crate::common::{ use crate::per_block_processing::errors::{BlockProcessingError, IntoWithIndex}; use crate::VerifySignatures; use types::consts::altair::{PARTICIPATION_FLAG_WEIGHTS, PROPOSER_WEIGHT, WEIGHT_DENOMINATOR}; +use types::validator::is_compounding_withdrawal_credential; pub fn process_operations>( state: &mut BeaconState, @@ -431,15 +432,47 @@ pub fn apply_deposit( let amount = deposit.data.amount; if let Some(index) = validator_index { - // Update the existing validator balance. - increase_balance(state, index as usize, amount)?; + // [Modified in Electra:EIP7251] + if let Ok(pending_balance_deposits) = state.pending_balance_deposits_mut() { + pending_balance_deposits.push(PendingBalanceDeposit { index, amount })?; + + let validator = state + .validators() + .get(index as usize) + .ok_or(BeaconStateError::UnknownValidator(index as usize))?; + + if is_compounding_withdrawal_credential(deposit.data.withdrawal_credentials, spec) + && validator.has_eth1_withdrawal_credential(spec) + && is_valid_deposit_signature(&deposit.data, spec).is_ok() + { + state.switch_to_compounding_validator(index as usize, spec)?; + } + } else { + // Update the existing validator balance. + increase_balance(state, index as usize, amount)?; + } } else { // The signature should be checked for new validators. Return early for a bad // signature. - if verify_deposit_signature(&deposit.data, spec).is_err() { + if is_valid_deposit_signature(&deposit.data, spec).is_err() { return Ok(()); } + let new_validator_index = state.validators().len(); + + // [Modified in Electra:EIP7251] + let (effective_balance, state_balance) = if state.fork_name_unchecked() >= ForkName::Electra + { + (0, 0) + } else { + ( + std::cmp::min( + amount.safe_sub(amount.safe_rem(spec.effective_balance_increment)?)?, + spec.max_effective_balance, + ), + amount, + ) + }; // Create a new validator. let validator = Validator { pubkey: deposit.data.pubkey, @@ -448,14 +481,11 @@ pub fn apply_deposit( activation_epoch: spec.far_future_epoch, exit_epoch: spec.far_future_epoch, withdrawable_epoch: spec.far_future_epoch, - effective_balance: std::cmp::min( - amount.safe_sub(amount.safe_rem(spec.effective_balance_increment)?)?, - spec.max_effective_balance, - ), + effective_balance, slashed: false, }; state.validators_mut().push(validator)?; - state.balances_mut().push(deposit.data.amount)?; + state.balances_mut().push(state_balance)?; // Altair or later initializations. if let Ok(previous_epoch_participation) = state.previous_epoch_participation_mut() { @@ -467,6 +497,14 @@ pub fn apply_deposit( if let Ok(inactivity_scores) = state.inactivity_scores_mut() { inactivity_scores.push(0)?; } + + // [New in Electra:EIP7251] + if let Ok(pending_balance_deposits) = state.pending_balance_deposits_mut() { + pending_balance_deposits.push(PendingBalanceDeposit { + index: new_validator_index as u64, + amount, + })?; + } } Ok(()) diff --git a/consensus/state_processing/src/per_block_processing/verify_deposit.rs b/consensus/state_processing/src/per_block_processing/verify_deposit.rs index a964f3b5740..c996e580a78 100644 --- a/consensus/state_processing/src/per_block_processing/verify_deposit.rs +++ b/consensus/state_processing/src/per_block_processing/verify_deposit.rs @@ -14,7 +14,7 @@ fn error(reason: DepositInvalid) -> BlockOperationError { /// Verify `Deposit.pubkey` signed `Deposit.signature`. /// /// Spec v0.12.1 -pub fn verify_deposit_signature(deposit_data: &DepositData, spec: &ChainSpec) -> Result<()> { +pub fn is_valid_deposit_signature(deposit_data: &DepositData, spec: &ChainSpec) -> Result<()> { let (public_key, signature, msg) = deposit_pubkey_signature_message(deposit_data, spec) .ok_or_else(|| error(DepositInvalid::BadBlsBytes))?; From 1d5f75582f166cec34fd2ed5bea9bd7584de7049 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 7 May 2024 10:25:53 -0400 Subject: [PATCH 011/227] add execution layer withdrawal request processing --- .../process_operations.rs | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 35a0c0ffadb..e4d89f5ef9e 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -509,3 +509,107 @@ pub fn apply_deposit( Ok(()) } + +pub fn process_execution_layer_withdrawal_requests( + state: &mut BeaconState, + requests: &[ExecutionLayerWithdrawalRequest], + spec: &ChainSpec, +) -> Result<(), BlockProcessingError> { + for request in requests { + let amount = request.amount; + let is_full_exit_request = amount == spec.full_exit_request_amount; + + // If partial withdrawal queue is full, only full exits are processed + if state.pending_partial_withdrawals()?.len() == E::pending_partial_withdrawals_limit() + && !is_full_exit_request + { + continue; + } + + // Verify pubkey exists + let index_opt = state.get_validator_index(&request.validator_pubkey)?; + let Some(index) = index_opt else { + continue; + }; + + let validator = state + .validators() + .get(index) + .ok_or(BeaconStateError::UnknownValidator(index))?; + + // Verify withdrawal credentials + let has_correct_credential = validator.has_execution_withdrawal_credential(spec); + let is_correct_source_address = validator + .withdrawal_credentials + .as_bytes() + .get(12..) + .ok_or(BeaconStateError::IndexNotSupported(12))? + == request.source_address.as_bytes(); + if !(has_correct_credential && is_correct_source_address) { + continue; + } + + // Verify the validator is active + if !validator.is_active_at(state.current_epoch()) { + continue; + } + + // Verify exit has not been initiated + if validator.exit_epoch != spec.far_future_epoch { + continue; + } + + // Verify the validator has been active long enough + if state.current_epoch() + < validator + .activation_epoch + .safe_add(spec.shard_committee_period)? + { + continue; + } + + let pending_balance_to_withdraw = state.get_pending_balance_to_withdraw(index)?; + if is_full_exit_request { + // Only exit validator if it has no pending withdrawals in the queue + if pending_balance_to_withdraw == 0 { + initiate_validator_exit(state, index, spec)? + } + continue; + } + + let balance = *state + .balances() + .get(index) + .ok_or(BeaconStateError::UnknownValidator(index))?; + let has_sufficient_effective_balance = + validator.effective_balance >= spec.min_activation_balance; + let has_excess_balance = balance + > spec + .min_activation_balance + .safe_add(pending_balance_to_withdraw)?; + + // Only allow partial withdrawals with compounding withdrawal credentials + if validator.has_compounding_withdrawal_credential(spec) + && has_sufficient_effective_balance + && has_excess_balance + { + let to_withdraw = std::cmp::min( + balance + .safe_sub(spec.min_activation_balance)? + .safe_sub(pending_balance_to_withdraw)?, + amount, + ); + let exit_queue_epoch = state.compute_exit_epoch_and_update_churn(to_withdraw, spec)?; + let withdrawable_epoch = + exit_queue_epoch.safe_add(spec.min_validator_withdrawability_delay)?; + state + .pending_partial_withdrawals_mut()? + .push(PendingPartialWithdrawal { + index: index as u64, + amount: to_withdraw, + withdrawable_epoch, + })?; + } + } + Ok(()) +} From 32357d8f0a7408f3378abe08620e37033dd57295 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 7 May 2024 11:07:12 -0400 Subject: [PATCH 012/227] process deposit receipts --- .../genesis/src/eth1_genesis_service.rs | 8 ++- consensus/state_processing/src/genesis.rs | 5 +- .../process_operations.rs | 53 ++++++++++++++----- 3 files changed, 50 insertions(+), 16 deletions(-) diff --git a/beacon_node/genesis/src/eth1_genesis_service.rs b/beacon_node/genesis/src/eth1_genesis_service.rs index 0ede74ba754..3e91504951c 100644 --- a/beacon_node/genesis/src/eth1_genesis_service.rs +++ b/beacon_node/genesis/src/eth1_genesis_service.rs @@ -432,8 +432,14 @@ impl Eth1GenesisService { // Such an optimization would only be useful in a scenario where `MIN_GENESIS_TIME` // is reached _prior_ to `MIN_ACTIVE_VALIDATOR_COUNT`. I suspect this won't be the // case for mainnet, so we defer this optimization. + let Deposit { proof, data } = deposit; + let proof = if PROOF_VERIFICATION { + Some(proof) + } else { + None + }; - apply_deposit(&mut state, &deposit, spec, PROOF_VERIFICATION) + apply_deposit(&mut state, data, proof, spec) .map_err(|e| format!("Error whilst processing deposit: {:?}", e)) })?; diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index a84f359389c..38006b83cac 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -32,12 +32,13 @@ pub fn initialize_beacon_state_from_eth1( let mut deposit_tree = DepositDataTree::create(&[], 0, DEPOSIT_TREE_DEPTH); - for deposit in deposits.iter() { + for deposit in deposits.into_iter() { deposit_tree .push_leaf(deposit.data.tree_hash_root()) .map_err(BlockProcessingError::MerkleTreeError)?; state.eth1_data_mut().deposit_root = deposit_tree.root(); - apply_deposit(&mut state, deposit, spec, true)?; + let Deposit { proof, data } = deposit; + apply_deposit(&mut state, data, Some(proof), spec)?; } process_activations(&mut state, spec)?; diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index e4d89f5ef9e..3ac12f43a1f 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -6,6 +6,7 @@ use crate::common::{ use crate::per_block_processing::errors::{BlockProcessingError, IntoWithIndex}; use crate::VerifySignatures; use types::consts::altair::{PARTICIPATION_FLAG_WEIGHTS, PROPOSER_WEIGHT, WEIGHT_DENOMINATOR}; +use types::typenum::U33; use types::validator::is_compounding_withdrawal_credential; pub fn process_operations>( @@ -403,22 +404,26 @@ pub fn process_deposits( // Update the state in series. for deposit in deposits { - apply_deposit(state, deposit, spec, false)?; + apply_deposit(state, deposit.data.clone(), None, spec)?; } Ok(()) } -/// Process a single deposit, optionally verifying its merkle proof. +/// Process a single deposit, verifying its merkle proof if provided. pub fn apply_deposit( state: &mut BeaconState, - deposit: &Deposit, + deposit_data: DepositData, + proof: Option>, spec: &ChainSpec, - verify_merkle_proof: bool, ) -> Result<(), BlockProcessingError> { let deposit_index = state.eth1_deposit_index() as usize; - if verify_merkle_proof { - verify_deposit_merkle_proof(state, deposit, state.eth1_deposit_index(), spec) + if let Some(proof) = proof { + let deposit = Deposit { + proof, + data: deposit_data.clone(), + }; + verify_deposit_merkle_proof(state, &deposit, state.eth1_deposit_index(), spec) .map_err(|e| e.into_with_index(deposit_index))?; } @@ -426,10 +431,10 @@ pub fn apply_deposit( // Get an `Option` where `u64` is the validator index if this deposit public key // already exists in the beacon_state. - let validator_index = get_existing_validator_index(state, &deposit.data.pubkey) + let validator_index = get_existing_validator_index(state, &deposit_data.pubkey) .map_err(|e| e.into_with_index(deposit_index))?; - let amount = deposit.data.amount; + let amount = deposit_data.amount; if let Some(index) = validator_index { // [Modified in Electra:EIP7251] @@ -441,9 +446,9 @@ pub fn apply_deposit( .get(index as usize) .ok_or(BeaconStateError::UnknownValidator(index as usize))?; - if is_compounding_withdrawal_credential(deposit.data.withdrawal_credentials, spec) + if is_compounding_withdrawal_credential(deposit_data.withdrawal_credentials, spec) && validator.has_eth1_withdrawal_credential(spec) - && is_valid_deposit_signature(&deposit.data, spec).is_ok() + && is_valid_deposit_signature(&deposit_data, spec).is_ok() { state.switch_to_compounding_validator(index as usize, spec)?; } @@ -454,7 +459,7 @@ pub fn apply_deposit( } else { // The signature should be checked for new validators. Return early for a bad // signature. - if is_valid_deposit_signature(&deposit.data, spec).is_err() { + if is_valid_deposit_signature(&deposit_data, spec).is_err() { return Ok(()); } @@ -475,8 +480,8 @@ pub fn apply_deposit( }; // Create a new validator. let validator = Validator { - pubkey: deposit.data.pubkey, - withdrawal_credentials: deposit.data.withdrawal_credentials, + pubkey: deposit_data.pubkey, + withdrawal_credentials: deposit_data.withdrawal_credentials, activation_eligibility_epoch: spec.far_future_epoch, activation_epoch: spec.far_future_epoch, exit_epoch: spec.far_future_epoch, @@ -613,3 +618,25 @@ pub fn process_execution_layer_withdrawal_requests( } Ok(()) } + +pub fn process_deposit_receipts( + state: &mut BeaconState, + receipts: &[DepositReceipt], + spec: &ChainSpec, +) -> Result<(), BlockProcessingError> { + for receipt in receipts { + // Set deposit receipt start index + if state.deposit_receipts_start_index()? == spec.unset_deposit_receipts_start_index { + *state.deposit_receipts_start_index_mut()? = receipt.index + } + let deposit_data = DepositData { + pubkey: receipt.pubkey, + withdrawal_credentials: receipt.withdrawal_credentials, + amount: receipt.amount, + signature: receipt.signature.clone().into(), + }; + apply_deposit(state, deposit_data, None, spec)? + } + + Ok(()) +} From c40bec93192cc21277dd52cb6d5c54cdfdd36da4 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 7 May 2024 14:01:44 -0400 Subject: [PATCH 013/227] add consolidation processing --- .../block_signature_verifier.rs | 22 +++ .../src/per_block_processing/errors.rs | 40 +++++ .../process_operations.rs | 159 ++++++++++++++++++ .../per_block_processing/signature_sets.rs | 39 ++++- consensus/types/src/beacon_state.rs | 1 + consensus/types/src/consolidation.rs | 4 +- consensus/types/src/validator.rs | 12 ++ 7 files changed, 274 insertions(+), 3 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs b/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs index 74477f5e481..a0c044219d5 100644 --- a/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs +++ b/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs @@ -171,6 +171,7 @@ where self.include_exits(block)?; self.include_sync_aggregate(block)?; self.include_bls_to_execution_changes(block)?; + self.include_consolidations(block)?; Ok(()) } @@ -359,6 +360,27 @@ where Ok(()) } + /// Includes all signatures in `self.block.body.consolidations` for verification. + pub fn include_consolidations>( + &mut self, + block: &'a SignedBeaconBlock, + ) -> Result<()> { + if let Ok(consolidations) = block.message().body().consolidations() { + self.sets.sets.reserve(consolidations.len()); + for consolidation in consolidations { + let set = consolidation_signature_set( + self.state, + self.get_pubkey.clone(), + consolidation, + self.spec, + )?; + + self.sets.push(set); + } + } + Ok(()) + } + /// Verify all the signatures that have been included in `self`, returning `true` if and only if /// all the signatures are valid. /// diff --git a/consensus/state_processing/src/per_block_processing/errors.rs b/consensus/state_processing/src/per_block_processing/errors.rs index 336895514f9..e3e15303f8d 100644 --- a/consensus/state_processing/src/per_block_processing/errors.rs +++ b/consensus/state_processing/src/per_block_processing/errors.rs @@ -89,6 +89,46 @@ pub enum BlockProcessingError { found: Hash256, }, WithdrawalCredentialsInvalid, + TooManyPendingConsolidations { + consolidations: usize, + limit: usize, + }, + ConsolidationChurnLimitTooLow { + churn_limit: u64, + minimum: u64, + }, + MatchingSourceTargetConsolidation { + index: u64, + }, + InactiveConsolidationSource { + index: u64, + current_epoch: Epoch, + }, + InactiveConsolidationTarget { + index: u64, + current_epoch: Epoch, + }, + SourceValidatorExiting { + index: u64, + }, + TargetValidatorExiting { + index: u64, + }, + FutureConsolidationEpoch { + current_epoch: Epoch, + consolidation_epoch: Epoch, + }, + NoSourceExecutionWithdrawalCredential { + index: u64, + }, + NoTargetExecutionWithdrawalCredential { + index: u64, + }, + MismatchedWithdrawalCredentials { + source_address: Address, + target_address: Address, + }, + InavlidConsolidationSignature, } impl From for BlockProcessingError { diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 3ac12f43a1f..7216f206a19 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -4,6 +4,7 @@ use crate::common::{ slash_validator, }; use crate::per_block_processing::errors::{BlockProcessingError, IntoWithIndex}; +use crate::signature_sets::consolidation_signature_set; use crate::VerifySignatures; use types::consts::altair::{PARTICIPATION_FLAG_WEIGHTS, PROPOSER_WEIGHT, WEIGHT_DENOMINATOR}; use types::typenum::U33; @@ -640,3 +641,161 @@ pub fn process_deposit_receipts( Ok(()) } + +pub fn process_consolidations( + state: &mut BeaconState, + consolidations: &[SignedConsolidation], + verify_signatures: VerifySignatures, + spec: &ChainSpec, +) -> Result<(), BlockProcessingError> { + // If the pending consolidations queue is full, no consolidations are allowed in the block + let pending_consolidations = state.pending_consolidations()?.len(); + let pending_consolidations_limit = E::pending_consolidations_limit(); + block_verify! { + pending_consolidations < pending_consolidations_limit, + BlockProcessingError::TooManyPendingConsolidations { + consolidations: pending_consolidations, + limit: pending_consolidations_limit + } + } + + // If there is too little available consolidation churn limit, no consolidations are allowed in the block + let churn_limit = state.get_consolidation_churn_limit(spec)?; + block_verify! { + churn_limit > spec.min_activation_balance, + BlockProcessingError::ConsolidationChurnLimitTooLow { + churn_limit, + minimum: spec.min_activation_balance + } + } + + for signed_consolidation in consolidations { + let consolidation = signed_consolidation.message.clone(); + + // Verify that source != target, so a consolidation cannot be used as an exit. + block_verify! { + consolidation.source_index != consolidation.target_index, + BlockProcessingError::MatchingSourceTargetConsolidation { + index: consolidation.source_index + } + } + + let source_validator = state + .validators() + .get(consolidation.source_index as usize) + .ok_or(BeaconStateError::UnknownValidator( + consolidation.source_index as usize, + ))?; + let target_validator = state + .validators() + .get(consolidation.target_index as usize) + .ok_or(BeaconStateError::UnknownValidator( + consolidation.target_index as usize, + ))?; + + // Verify the source and the target are active + let current_epoch = state.current_epoch(); + block_verify! { + source_validator.is_active_at(current_epoch), + BlockProcessingError::InactiveConsolidationSource{ + index: consolidation.source_index, + current_epoch + } + } + block_verify! { + target_validator.is_active_at(current_epoch), + BlockProcessingError::InactiveConsolidationTarget{ + index: consolidation.target_index, + current_epoch + } + } + + // Verify exits for source and target have not been initiated + block_verify! { + source_validator.exit_epoch == spec.far_future_epoch, + BlockProcessingError::SourceValidatorExiting{ + index: consolidation.source_index, + } + } + block_verify! { + target_validator.exit_epoch == spec.far_future_epoch, + BlockProcessingError::TargetValidatorExiting{ + index: consolidation.target_index, + } + } + + // Consolidations must specify an epoch when they become valid; they are not valid before then + block_verify! { + current_epoch >= consolidation.epoch, + BlockProcessingError::FutureConsolidationEpoch { + current_epoch, + consolidation_epoch: consolidation.epoch + } + } + + // Verify the source and the target have Execution layer withdrawal credentials + block_verify! { + source_validator.has_execution_withdrawal_credential(spec), + BlockProcessingError::NoSourceExecutionWithdrawalCredential { + index: consolidation.source_index, + } + } + block_verify! { + target_validator.has_execution_withdrawal_credential(spec), + BlockProcessingError::NoTargetExecutionWithdrawalCredential { + index: consolidation.target_index, + } + } + + // Verify the same withdrawal address + let source_address = source_validator + .get_execution_withdrawal_address(spec) + .ok_or(BeaconStateError::NonExecutionAddresWithdrawalCredential)?; + let target_address = target_validator + .get_execution_withdrawal_address(spec) + .ok_or(BeaconStateError::NonExecutionAddresWithdrawalCredential)?; + block_verify! { + source_address == target_address, + BlockProcessingError::MismatchedWithdrawalCredentials { + source_address, + target_address + } + } + + if verify_signatures.is_true() { + let signature_set = consolidation_signature_set( + state, + |i| get_pubkey_from_state(state, i), + signed_consolidation, + spec, + )?; + block_verify! { + signature_set.verify(), + BlockProcessingError::InavlidConsolidationSignature + } + } + let exit_epoch = state.compute_consolidation_epoch_and_update_churn( + source_validator.effective_balance, + spec, + )?; + let source_validator = state + .validators_mut() + .get_mut(consolidation.source_index as usize) + .ok_or(BeaconStateError::UnknownValidator( + consolidation.source_index as usize, + ))?; + // Initiate source validator exit and append pending consolidation + source_validator.exit_epoch = exit_epoch; + source_validator.withdrawable_epoch = source_validator + .exit_epoch + .safe_add(spec.min_validator_withdrawability_delay)?; + state + .pending_consolidations_mut()? + .push(PendingConsolidation { + source_index: consolidation.source_index, + target_index: consolidation.target_index, + })?; + } + + Ok(()) +} diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index a179583556f..65b14714f86 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -11,8 +11,8 @@ use types::{ BeaconStateError, ChainSpec, DepositData, Domain, Epoch, EthSpec, Fork, Hash256, InconsistentFork, IndexedAttestation, IndexedAttestationRef, ProposerSlashing, PublicKey, PublicKeyBytes, Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockHeader, - SignedBlsToExecutionChange, SignedContributionAndProof, SignedRoot, SignedVoluntaryExit, - SigningData, Slot, SyncAggregate, SyncAggregatorSelectionData, Unsigned, + SignedBlsToExecutionChange, SignedConsolidation, SignedContributionAndProof, SignedRoot, + SignedVoluntaryExit, SigningData, Slot, SyncAggregate, SyncAggregatorSelectionData, Unsigned, }; pub type Result = std::result::Result; @@ -665,3 +665,38 @@ where message, ))) } + +/// Returns two signature sets, one for the source and one for the target validator +/// in the `SignedConsolidation`. +pub fn consolidation_signature_set<'a, E, F>( + state: &'a BeaconState, + get_pubkey: F, + consolidation: &'a SignedConsolidation, + spec: &'a ChainSpec, +) -> Result> +where + E: EthSpec, + F: Fn(usize) -> Option>, +{ + let source_index = consolidation.message.source_index as usize; + let target_index = consolidation.message.target_index as usize; + + let domain = spec.get_domain( + consolidation.message.epoch, + Domain::Consolidation, + &state.fork(), + state.genesis_validators_root(), + ); + + let message = consolidation.message.signing_root(domain); + let source_pubkey = + get_pubkey(source_index).ok_or(Error::ValidatorUnknown(source_index as u64))?; + let target_pubkey = + get_pubkey(target_index).ok_or(Error::ValidatorUnknown(target_index as u64))?; + + Ok(SignatureSet::multiple_pubkeys( + &consolidation.signature, + vec![source_pubkey, target_pubkey], + message, + )) +} diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index dda87c5dd50..a1601a5d1ff 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -160,6 +160,7 @@ pub enum Error { InvalidFlagIndex(usize), MerkleTreeError(merkle_proof::MerkleTreeError), PartialWithdrawalCountInvalid(usize), + NonExecutionAddresWithdrawalCredential, } /// Control whether an epoch-indexed field can be indexed at the next epoch or not. diff --git a/consensus/types/src/consolidation.rs b/consensus/types/src/consolidation.rs index 09a2d4bb0c3..6cc4aa90f27 100644 --- a/consensus/types/src/consolidation.rs +++ b/consensus/types/src/consolidation.rs @@ -1,5 +1,5 @@ -use crate::test_utils::TestRandom; use crate::Epoch; +use crate::{test_utils::TestRandom, SignedRoot}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -27,6 +27,8 @@ pub struct Consolidation { pub epoch: Epoch, } +impl SignedRoot for Consolidation {} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index 3287942dfe7..86243083f06 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -142,6 +142,18 @@ impl Validator { .flatten() } + /// Get the execution withdrawal address if this validator has one initialized. + pub fn get_execution_withdrawal_address(&self, spec: &ChainSpec) -> Option
{ + self.has_execution_withdrawal_credential(spec) + .then(|| { + self.withdrawal_credentials + .as_bytes() + .get(12..) + .map(Address::from_slice) + }) + .flatten() + } + /// Changes withdrawal credentials to the provided eth1 execution address. /// /// WARNING: this function does NO VALIDATION - it just does it! From 31955c2e7f8dc1f9e67a9116b43ecd1b4dc0d032 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 7 May 2024 14:52:16 -0400 Subject: [PATCH 014/227] update process operations function --- .../process_operations.rs | 12 ++ consensus/types/src/payload.rs | 151 +++++++++++++++++- 2 files changed, 161 insertions(+), 2 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 7216f206a19..9341ca811bc 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -39,6 +39,18 @@ pub fn process_operations>( process_bls_to_execution_changes(state, bls_to_execution_changes, verify_signatures, spec)?; } + if state.fork_name_unchecked() >= ForkName::Electra { + let requests = block_body.execution_payload()?.withdrawal_requests()?; + if let Some(requests) = requests { + process_execution_layer_withdrawal_requests(state, &requests, spec)?; + } + let receipts = block_body.execution_payload()?.deposit_receipts()?; + if let Some(receipts) = receipts { + process_deposit_receipts(state, &receipts, spec)?; + } + process_consolidations(state, block_body.consolidations()?, verify_signatures, spec)?; + } + Ok(()) } diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 80a70c171f5..644d401ec7e 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -39,6 +39,15 @@ pub trait ExecPayload: Debug + Clone + PartialEq + Hash + TreeHash + /// fork-specific fields fn withdrawals_root(&self) -> Result; fn blob_gas_used(&self) -> Result; + fn withdrawal_requests( + &self, + ) -> Result< + Option>, + Error, + >; + fn deposit_receipts( + &self, + ) -> Result>, Error>; /// Is this a default payload with 0x0 roots for transactions and withdrawals? fn is_default_with_zero_roots(&self) -> bool; @@ -278,6 +287,35 @@ impl ExecPayload for FullPayload { } } + fn withdrawal_requests( + &self, + ) -> Result< + Option>, + Error, + > { + match self { + FullPayload::Bellatrix(_) | FullPayload::Capella(_) | FullPayload::Deneb(_) => { + Err(Error::IncorrectStateVariant) + } + FullPayload::Electra(inner) => { + Ok(Some(inner.execution_payload.withdrawal_requests.clone())) + } + } + } + + fn deposit_receipts( + &self, + ) -> Result>, Error> { + match self { + FullPayload::Bellatrix(_) | FullPayload::Capella(_) | FullPayload::Deneb(_) => { + Err(Error::IncorrectStateVariant) + } + FullPayload::Electra(inner) => { + Ok(Some(inner.execution_payload.deposit_receipts.clone())) + } + } + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_full_payload_ref!(&'a _, self.to_ref(), move |payload, cons| { cons(payload); @@ -410,6 +448,35 @@ impl<'b, E: EthSpec> ExecPayload for FullPayloadRef<'b, E> { } } + fn withdrawal_requests( + &self, + ) -> Result< + Option>, + Error, + > { + match self { + FullPayloadRef::Bellatrix(_) + | FullPayloadRef::Capella(_) + | FullPayloadRef::Deneb(_) => Err(Error::IncorrectStateVariant), + FullPayloadRef::Electra(inner) => { + Ok(Some(inner.execution_payload.withdrawal_requests.clone())) + } + } + } + + fn deposit_receipts( + &self, + ) -> Result>, Error> { + match self { + FullPayloadRef::Bellatrix(_) + | FullPayloadRef::Capella(_) + | FullPayloadRef::Deneb(_) => Err(Error::IncorrectStateVariant), + FullPayloadRef::Electra(inner) => { + Ok(Some(inner.execution_payload.deposit_receipts.clone())) + } + } + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_full_payload_ref!(&'a _, self, move |payload, cons| { cons(payload); @@ -590,6 +657,21 @@ impl ExecPayload for BlindedPayload { } } + fn withdrawal_requests( + &self, + ) -> Result< + Option>, + Error, + > { + Ok(None) + } + + fn deposit_receipts( + &self, + ) -> Result>, Error> { + Ok(None) + } + fn is_default_with_zero_roots(&self) -> bool { self.to_ref().is_default_with_zero_roots() } @@ -691,6 +773,21 @@ impl<'b, E: EthSpec> ExecPayload for BlindedPayloadRef<'b, E> { } } + fn withdrawal_requests( + &self, + ) -> Result< + Option>, + Error, + > { + Ok(None) + } + + fn deposit_receipts( + &self, + ) -> Result>, Error> { + Ok(None) + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_blinded_payload_ref!(&'b _, self, move |payload, cons| { cons(payload); @@ -717,7 +814,9 @@ macro_rules! impl_exec_payload_common { $is_default_with_empty_roots:block, $f:block, $g:block, - $h:block) => { + $h:block, + $i:block, + $j:block) => { impl ExecPayload for $wrapper_type { fn block_type() -> BlockType { BlockType::$block_type_variant @@ -780,6 +879,23 @@ macro_rules! impl_exec_payload_common { let h = $h; h(self) } + + fn withdrawal_requests( + &self, + ) -> Result< + Option>, + Error, + > { + let i = $i; + i(self) + } + + fn deposit_receipts( + &self, + ) -> Result>, Error> { + let j = $j; + j(self) + } } impl From<$wrapped_type> for $wrapper_type { @@ -825,7 +941,9 @@ macro_rules! impl_exec_payload_for_fork { wrapper_ref_type.blob_gas_used() }; c - } + }, + { |_| { Ok(None) } }, + { |_| { Ok(None) } } ); impl TryInto<$wrapper_type_header> for BlindedPayload { @@ -912,6 +1030,35 @@ macro_rules! impl_exec_payload_for_fork { wrapper_ref_type.blob_gas_used() }; c + }, + { + let c: for<'a> fn( + &'a $wrapper_type_full, + ) -> Result< + Option< + VariableList< + ExecutionLayerWithdrawalRequest, + E::MaxWithdrawalRequestsPerPayload, + >, + >, + Error, + > = |payload: &$wrapper_type_full| { + let wrapper_ref_type = FullPayloadRef::$fork_variant(&payload); + wrapper_ref_type.withdrawal_requests() + }; + c + }, + { + let c: for<'a> fn( + &'a $wrapper_type_full, + ) -> Result< + Option>, + Error, + > = |payload: &$wrapper_type_full| { + let wrapper_ref_type = FullPayloadRef::$fork_variant(&payload); + wrapper_ref_type.deposit_receipts() + }; + c } ); From 75ab913a3a9db0f332a7bec2eb8778b9452498de Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 7 May 2024 15:32:07 -0400 Subject: [PATCH 015/227] exit updates --- .../src/common/initiate_validator_exit.rs | 26 ++++++++++++------- .../src/per_block_processing/errors.rs | 6 ++++- .../src/per_block_processing/verify_exit.rs | 11 ++++++++ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/consensus/state_processing/src/common/initiate_validator_exit.rs b/consensus/state_processing/src/common/initiate_validator_exit.rs index a40a9dfd398..8874e9ed4b8 100644 --- a/consensus/state_processing/src/common/initiate_validator_exit.rs +++ b/consensus/state_processing/src/common/initiate_validator_exit.rs @@ -19,16 +19,22 @@ pub fn initiate_validator_exit( state.build_exit_cache(spec)?; // Compute exit queue epoch - let delayed_epoch = state.compute_activation_exit_epoch(state.current_epoch(), spec)?; - let mut exit_queue_epoch = state - .exit_cache() - .max_epoch()? - .map_or(delayed_epoch, |epoch| max(epoch, delayed_epoch)); - let exit_queue_churn = state.exit_cache().get_churn_at(exit_queue_epoch)?; - - if exit_queue_churn >= state.get_validator_churn_limit(spec)? { - exit_queue_epoch.safe_add_assign(1)?; - } + let exit_queue_epoch = if state.fork_name_unchecked() >= ForkName::Electra { + let effective_balance = state.get_validator(index)?.effective_balance; + state.compute_exit_epoch_and_update_churn(effective_balance, spec)? + } else { + let delayed_epoch = state.compute_activation_exit_epoch(state.current_epoch(), spec)?; + let mut exit_queue_epoch = state + .exit_cache() + .max_epoch()? + .map_or(delayed_epoch, |epoch| max(epoch, delayed_epoch)); + let exit_queue_churn = state.exit_cache().get_churn_at(exit_queue_epoch)?; + + if exit_queue_churn >= state.get_validator_churn_limit(spec)? { + exit_queue_epoch.safe_add_assign(1)?; + } + exit_queue_epoch + }; let validator = state.get_validator_cow(index)?; diff --git a/consensus/state_processing/src/per_block_processing/errors.rs b/consensus/state_processing/src/per_block_processing/errors.rs index e3e15303f8d..46665a88392 100644 --- a/consensus/state_processing/src/per_block_processing/errors.rs +++ b/consensus/state_processing/src/per_block_processing/errors.rs @@ -451,7 +451,10 @@ pub enum ExitInvalid { /// The specified validator has already initiated exit. AlreadyInitiatedExit(u64), /// The exit is for a future epoch. - FutureEpoch { state: Epoch, exit: Epoch }, + FutureEpoch { + state: Epoch, + exit: Epoch, + }, /// The validator has not been active for long enough. TooYoungToExit { current_epoch: Epoch, @@ -462,6 +465,7 @@ pub enum ExitInvalid { /// There was an error whilst attempting to get a set of signatures. The signatures may have /// been invalid or an internal error occurred. SignatureSetError(SignatureSetError), + PendingWithdrawalInQueue(u64), } #[derive(Debug, PartialEq, Clone)] diff --git a/consensus/state_processing/src/per_block_processing/verify_exit.rs b/consensus/state_processing/src/per_block_processing/verify_exit.rs index fc258d38298..dea17dbc0c4 100644 --- a/consensus/state_processing/src/per_block_processing/verify_exit.rs +++ b/consensus/state_processing/src/per_block_processing/verify_exit.rs @@ -79,5 +79,16 @@ pub fn verify_exit( ); } + // [New in Electra:EIP7251] + // Only exit validator if it has no pending withdrawals in the queue + if let Ok(pending_balance_to_withdraw) = + state.get_pending_balance_to_withdraw(exit.validator_index as usize) + { + verify!( + pending_balance_to_withdraw == 0, + ExitInvalid::PendingWithdrawalInQueue(exit.validator_index) + ); + } + Ok(()) } From 5728f780326500da7100376eb45e74b1a291cce8 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 7 May 2024 16:02:14 -0400 Subject: [PATCH 016/227] clean up --- .../src/per_block_processing.rs | 22 +++------- .../process_operations.rs | 40 +++++-------------- consensus/types/src/beacon_state.rs | 8 ++++ 3 files changed, 22 insertions(+), 48 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 43414f70859..0f67dd95f85 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -521,16 +521,8 @@ pub fn get_expected_withdrawals( break; } - let withdrawal_balance = state - .balances() - .get(withdrawal.index as usize) - .copied() - .ok_or(BeaconStateError::BalancesOutOfBounds( - withdrawal.index as usize, - ))?; - let validator = state.validators().get(withdrawal.index as usize).ok_or( - BeaconStateError::UnknownValidator(withdrawal.index as usize), - )?; + let withdrawal_balance = state.get_balance(withdrawal.index as usize)?; + let validator = state.get_validator(withdrawal.index as usize)?; let has_sufficient_effective_balance = validator.effective_balance >= spec.min_activation_balance; @@ -547,13 +539,9 @@ pub fn get_expected_withdrawals( withdrawals.push(Withdrawal { index: withdrawal_index, validator_index: withdrawal.index, - address: Address::from_slice( - validator - .withdrawal_credentials - .0 - .get(12..) - .ok_or(BeaconStateError::IndexNotSupported(12))?, - ), + address: validator + .get_execution_withdrawal_address(spec) + .ok_or(BeaconStateError::NonExecutionAddresWithdrawalCredential)?, amount: withdrawable_balance, }); withdrawal_index.safe_add_assign(1)?; diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 9341ca811bc..aba184b8043 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -550,19 +550,15 @@ pub fn process_execution_layer_withdrawal_requests( continue; }; - let validator = state - .validators() - .get(index) - .ok_or(BeaconStateError::UnknownValidator(index))?; + let validator = state.get_validator(index)?; // Verify withdrawal credentials let has_correct_credential = validator.has_execution_withdrawal_credential(spec); let is_correct_source_address = validator - .withdrawal_credentials - .as_bytes() - .get(12..) - .ok_or(BeaconStateError::IndexNotSupported(12))? - == request.source_address.as_bytes(); + .get_execution_withdrawal_address(spec) + .ok_or(BeaconStateError::NonExecutionAddresWithdrawalCredential)? + == request.source_address; + if !(has_correct_credential && is_correct_source_address) { continue; } @@ -595,10 +591,7 @@ pub fn process_execution_layer_withdrawal_requests( continue; } - let balance = *state - .balances() - .get(index) - .ok_or(BeaconStateError::UnknownValidator(index))?; + let balance = state.get_balance(index)?; let has_sufficient_effective_balance = validator.effective_balance >= spec.min_activation_balance; let has_excess_balance = balance @@ -692,18 +685,8 @@ pub fn process_consolidations( } } - let source_validator = state - .validators() - .get(consolidation.source_index as usize) - .ok_or(BeaconStateError::UnknownValidator( - consolidation.source_index as usize, - ))?; - let target_validator = state - .validators() - .get(consolidation.target_index as usize) - .ok_or(BeaconStateError::UnknownValidator( - consolidation.target_index as usize, - ))?; + let source_validator = state.get_validator(consolidation.source_index as usize)?; + let target_validator = state.get_validator(consolidation.target_index as usize)?; // Verify the source and the target are active let current_epoch = state.current_epoch(); @@ -790,12 +773,7 @@ pub fn process_consolidations( source_validator.effective_balance, spec, )?; - let source_validator = state - .validators_mut() - .get_mut(consolidation.source_index as usize) - .ok_or(BeaconStateError::UnknownValidator( - consolidation.source_index as usize, - ))?; + let source_validator = state.get_validator_mut(consolidation.source_index as usize)?; // Initiate source validator exit and append pending consolidation source_validator.exit_epoch = exit_epoch; source_validator.withdrawable_epoch = source_validator diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index a1601a5d1ff..0ff2a24ca01 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -1469,6 +1469,14 @@ impl BeaconState { } } + /// Get the balance of a single validator. + pub fn get_balance(&self, validator_index: usize) -> Result { + self.balances() + .get(validator_index) + .ok_or(Error::BalancesOutOfBounds(validator_index)) + .copied() + } + /// Get a mutable reference to the balance of a single validator. pub fn get_balance_mut(&mut self, validator_index: usize) -> Result<&mut u64, Error> { self.balances_mut() From 8517236aeda6d1c0a09fd5996b718fae68272b6b Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 7 May 2024 16:24:14 -0400 Subject: [PATCH 017/227] update slash_validator --- .../state_processing/src/common/slash_validator.rs | 4 ++-- consensus/types/src/chain_spec.rs | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/consensus/state_processing/src/common/slash_validator.rs b/consensus/state_processing/src/common/slash_validator.rs index 520b58a8af3..80d857cc009 100644 --- a/consensus/state_processing/src/common/slash_validator.rs +++ b/consensus/state_processing/src/common/slash_validator.rs @@ -53,8 +53,8 @@ pub fn slash_validator( // Apply proposer and whistleblower rewards let proposer_index = ctxt.get_proposer_index(state, spec)? as usize; let whistleblower_index = opt_whistleblower_index.unwrap_or(proposer_index); - let whistleblower_reward = - validator_effective_balance.safe_div(spec.whistleblower_reward_quotient)?; + let whistleblower_reward = validator_effective_balance + .safe_div(spec.whistleblower_reward_quotient_for_state(state))?; let proposer_reward = match state { BeaconState::Base(_) => whistleblower_reward.safe_div(spec.proposer_reward_quotient)?, BeaconState::Altair(_) diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index b0346a14ef8..8ffafe6266d 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -387,6 +387,19 @@ impl ChainSpec { } } + /// For a given `BeaconState`, return the whistleblower reward quotient associated with its variant. + pub fn whistleblower_reward_quotient_for_state( + &self, + state: &BeaconState, + ) -> u64 { + let fork_name = state.fork_name_unchecked(); + if fork_name >= ForkName::Electra { + self.whistleblower_reward_quotient_electra + } else { + self.whistleblower_reward_quotient + } + } + /// Returns a full `Fork` struct for a given epoch. pub fn fork_at_epoch(&self, epoch: Epoch) -> Fork { let current_fork_name = self.fork_name_at_epoch(epoch); From befaf09aeb65cd2da48e49089eb2827af0bc8f5d Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Mon, 6 May 2024 15:48:46 -0700 Subject: [PATCH 018/227] Add new engine api methods --- beacon_node/execution_layer/src/engine_api.rs | 11 +- .../execution_layer/src/engine_api/http.rs | 52 +++++++-- .../src/engine_api/json_structures.rs | 102 +++++++++++++++++- .../execution_layer/src/test_utils/mod.rs | 2 + 4 files changed, 155 insertions(+), 12 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index ce1e0fec5dd..6f6ec8b8947 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -3,7 +3,8 @@ use crate::http::{ ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_FORKCHOICE_UPDATED_V3, ENGINE_GET_CLIENT_VERSION_V1, ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, - ENGINE_GET_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, + ENGINE_GET_PAYLOAD_V3, ENGINE_GET_PAYLOAD_V4, ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, + ENGINE_NEW_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V4, }; use eth2::types::{ BlobsBundle, SsePayloadAttributes, SsePayloadAttributesV1, SsePayloadAttributesV2, @@ -652,6 +653,7 @@ pub struct EngineCapabilities { pub new_payload_v1: bool, pub new_payload_v2: bool, pub new_payload_v3: bool, + pub new_payload_v4: bool, pub forkchoice_updated_v1: bool, pub forkchoice_updated_v2: bool, pub forkchoice_updated_v3: bool, @@ -660,6 +662,7 @@ pub struct EngineCapabilities { pub get_payload_v1: bool, pub get_payload_v2: bool, pub get_payload_v3: bool, + pub get_payload_v4: bool, pub get_client_version_v1: bool, } @@ -675,6 +678,9 @@ impl EngineCapabilities { if self.new_payload_v3 { response.push(ENGINE_NEW_PAYLOAD_V3); } + if self.new_payload_v4 { + response.push(ENGINE_NEW_PAYLOAD_V4); + } if self.forkchoice_updated_v1 { response.push(ENGINE_FORKCHOICE_UPDATED_V1); } @@ -699,6 +705,9 @@ impl EngineCapabilities { if self.get_payload_v3 { response.push(ENGINE_GET_PAYLOAD_V3); } + if self.get_payload_v4 { + response.push(ENGINE_GET_PAYLOAD_V4); + } if self.get_client_version_v1 { response.push(ENGINE_GET_CLIENT_VERSION_V1); } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 93705a16925..e81027340f6 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -34,11 +34,13 @@ pub const ETH_SYNCING_TIMEOUT: Duration = Duration::from_secs(1); pub const ENGINE_NEW_PAYLOAD_V1: &str = "engine_newPayloadV1"; pub const ENGINE_NEW_PAYLOAD_V2: &str = "engine_newPayloadV2"; pub const ENGINE_NEW_PAYLOAD_V3: &str = "engine_newPayloadV3"; +pub const ENGINE_NEW_PAYLOAD_V4: &str = "engine_newPayloadV4"; pub const ENGINE_NEW_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(8); pub const ENGINE_GET_PAYLOAD_V1: &str = "engine_getPayloadV1"; pub const ENGINE_GET_PAYLOAD_V2: &str = "engine_getPayloadV2"; pub const ENGINE_GET_PAYLOAD_V3: &str = "engine_getPayloadV3"; +pub const ENGINE_GET_PAYLOAD_V4: &str = "engine_getPayloadV4"; pub const ENGINE_GET_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(2); pub const ENGINE_FORKCHOICE_UPDATED_V1: &str = "engine_forkchoiceUpdatedV1"; @@ -66,9 +68,11 @@ pub static LIGHTHOUSE_CAPABILITIES: &[&str] = &[ ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, + ENGINE_NEW_PAYLOAD_V4, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, ENGINE_GET_PAYLOAD_V3, + ENGINE_GET_PAYLOAD_V4, ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_FORKCHOICE_UPDATED_V3, @@ -833,7 +837,7 @@ impl HttpJsonRpc { Ok(response.into()) } - pub async fn new_payload_v3_electra( + pub async fn new_payload_v4_electra( &self, new_payload_request_electra: NewPayloadRequestElectra<'_, E>, ) -> Result { @@ -845,7 +849,7 @@ impl HttpJsonRpc { let response: JsonPayloadStatusV1 = self .rpc_request( - ENGINE_NEW_PAYLOAD_V3, + ENGINE_NEW_PAYLOAD_V4, params, ENGINE_NEW_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, ) @@ -929,10 +933,40 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V3(response).into()) } + ForkName::Base + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + // TODO(pawan): make sure electra should not support getpayloadv3 + | ForkName::Electra => Err(Error::UnsupportedForkVariant(format!( + "called get_payload_v3 with {}", + fork_name + ))), + } + } + + pub async fn get_payload_v4( + &self, + fork_name: ForkName, + payload_id: PayloadId, + ) -> Result, Error> { + let params = json!([JsonPayloadIdRequest::from(payload_id)]); + + match fork_name { + ForkName::Deneb => { + let response: JsonGetPayloadResponseV3 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V4, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + Ok(JsonGetPayloadResponse::V3(response).into()) + } ForkName::Electra => { let response: JsonGetPayloadResponseV4 = self .rpc_request( - ENGINE_GET_PAYLOAD_V3, + ENGINE_GET_PAYLOAD_V4, params, ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, ) @@ -940,7 +974,7 @@ impl HttpJsonRpc { Ok(JsonGetPayloadResponse::V4(response).into()) } ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => Err( - Error::UnsupportedForkVariant(format!("called get_payload_v3 with {}", fork_name)), + Error::UnsupportedForkVariant(format!("called get_payload_v4 with {}", fork_name)), ), } } @@ -1067,6 +1101,7 @@ impl HttpJsonRpc { new_payload_v1: capabilities.contains(ENGINE_NEW_PAYLOAD_V1), new_payload_v2: capabilities.contains(ENGINE_NEW_PAYLOAD_V2), new_payload_v3: capabilities.contains(ENGINE_NEW_PAYLOAD_V3), + new_payload_v4: capabilities.contains(ENGINE_NEW_PAYLOAD_V4), forkchoice_updated_v1: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V1), forkchoice_updated_v2: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V2), forkchoice_updated_v3: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V3), @@ -1077,6 +1112,7 @@ impl HttpJsonRpc { get_payload_v1: capabilities.contains(ENGINE_GET_PAYLOAD_V1), get_payload_v2: capabilities.contains(ENGINE_GET_PAYLOAD_V2), get_payload_v3: capabilities.contains(ENGINE_GET_PAYLOAD_V3), + get_payload_v4: capabilities.contains(ENGINE_GET_PAYLOAD_V4), get_client_version_v1: capabilities.contains(ENGINE_GET_CLIENT_VERSION_V1), }) } @@ -1200,7 +1236,7 @@ impl HttpJsonRpc { } NewPayloadRequest::Electra(new_payload_request_electra) => { if engine_capabilities.new_payload_v3 { - self.new_payload_v3_electra(new_payload_request_electra) + self.new_payload_v4_electra(new_payload_request_electra) .await } else { Err(Error::RequiredMethodUnsupported("engine_newPayloadV3")) @@ -1221,14 +1257,16 @@ impl HttpJsonRpc { ForkName::Bellatrix | ForkName::Capella => { if engine_capabilities.get_payload_v2 { self.get_payload_v2(fork_name, payload_id).await - } else if engine_capabilities.new_payload_v1 { + } else if engine_capabilities.get_payload_v1 { self.get_payload_v1(payload_id).await } else { Err(Error::RequiredMethodUnsupported("engine_getPayload")) } } ForkName::Deneb | ForkName::Electra => { - if engine_capabilities.get_payload_v3 { + if engine_capabilities.get_payload_v4 { + self.get_payload_v4(fork_name, payload_id).await + } else if engine_capabilities.get_payload_v3 { self.get_payload_v3(fork_name, payload_id).await } else { Err(Error::RequiredMethodUnsupported("engine_getPayloadV3")) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 50d3519e129..822a3802a85 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -4,7 +4,10 @@ use strum::EnumString; use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::BlobsList; -use types::{FixedVector, Unsigned}; +use types::{ + DepositReceipt, ExecutionLayerWithdrawalRequest, FixedVector, PublicKeyBytes, Signature, + Unsigned, +}; #[derive(Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -101,6 +104,11 @@ pub struct JsonExecutionPayload { #[superstruct(only(V3, V4))] #[serde(with = "serde_utils::u64_hex_be")] pub excess_blob_gas: u64, + #[superstruct(only(V4))] + pub deposit_requests: VariableList, + #[superstruct(only(V4))] + pub withdrawal_requests: + VariableList, } impl From> for JsonExecutionPayloadV1 { @@ -203,6 +211,18 @@ impl From> for JsonExecutionPayloadV4 .into(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, + deposit_requests: payload + .deposit_receipts + .into_iter() + .map(Into::into) + .collect::>() + .into(), + withdrawal_requests: payload + .withdrawal_requests + .into_iter() + .map(Into::into) + .collect::>() + .into(), } } } @@ -319,9 +339,18 @@ impl From> for ExecutionPayloadElectra .into(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, - // TODO(electra) - deposit_receipts: Default::default(), - withdrawal_requests: Default::default(), + deposit_receipts: payload + .deposit_requests + .into_iter() + .map(Into::into) + .collect::>() + .into(), + withdrawal_requests: payload + .withdrawal_requests + .into_iter() + .map(Into::into) + .collect::>() + .into(), } } } @@ -783,3 +812,68 @@ impl TryFrom for ClientVersionV1 { }) } } + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct JsonDepositRequest { + pub pubkey: PublicKeyBytes, + pub withdrawal_credentials: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub amount: u64, + pub signature: Signature, + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, +} + +impl From for JsonDepositRequest { + fn from(deposit: DepositReceipt) -> Self { + Self { + pubkey: deposit.pubkey, + withdrawal_credentials: deposit.withdrawal_credentials, + amount: deposit.amount, + signature: deposit.signature, + index: deposit.index, + } + } +} + +impl From for DepositReceipt { + fn from(json_deposit: JsonDepositRequest) -> Self { + Self { + pubkey: json_deposit.pubkey, + withdrawal_credentials: json_deposit.withdrawal_credentials, + amount: json_deposit.amount, + signature: json_deposit.signature, + index: json_deposit.index, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct JsonWithdrawalRequest { + pub source_address: Address, + pub validator_public_key: PublicKeyBytes, + #[serde(with = "serde_utils::quoted_u64")] + pub amount: u64, +} + +impl From for JsonWithdrawalRequest { + fn from(withdrawal_request: ExecutionLayerWithdrawalRequest) -> Self { + Self { + source_address: withdrawal_request.source_address, + validator_public_key: withdrawal_request.validator_pubkey, + amount: withdrawal_request.amount, + } + } +} + +impl From for ExecutionLayerWithdrawalRequest { + fn from(json_withdrawal_request: JsonWithdrawalRequest) -> Self { + Self { + source_address: json_withdrawal_request.source_address, + validator_pubkey: json_withdrawal_request.validator_public_key, + amount: json_withdrawal_request.amount, + } + } +} diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index a6d47995af8..7b00ca9fbc3 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -43,6 +43,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { new_payload_v1: true, new_payload_v2: true, new_payload_v3: true, + new_payload_v4: true, forkchoice_updated_v1: true, forkchoice_updated_v2: true, forkchoice_updated_v3: true, @@ -51,6 +52,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { get_payload_v1: true, get_payload_v2: true, get_payload_v3: true, + get_payload_v4: true, get_client_version_v1: true, }; From fffa130aa7654dec15769feab8cd55f986fc6d73 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 7 May 2024 14:42:51 -0700 Subject: [PATCH 019/227] Fix the versioning of v4 requests --- .../execution_layer/src/engine_api/http.rs | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index e81027340f6..aea87dd7d41 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -937,7 +937,6 @@ impl HttpJsonRpc { | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella - // TODO(pawan): make sure electra should not support getpayloadv3 | ForkName::Electra => Err(Error::UnsupportedForkVariant(format!( "called get_payload_v3 with {}", fork_name @@ -953,16 +952,6 @@ impl HttpJsonRpc { let params = json!([JsonPayloadIdRequest::from(payload_id)]); match fork_name { - ForkName::Deneb => { - let response: JsonGetPayloadResponseV3 = self - .rpc_request( - ENGINE_GET_PAYLOAD_V4, - params, - ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, - ) - .await?; - Ok(JsonGetPayloadResponse::V3(response).into()) - } ForkName::Electra => { let response: JsonGetPayloadResponseV4 = self .rpc_request( @@ -973,9 +962,14 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V4(response).into()) } - ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => Err( - Error::UnsupportedForkVariant(format!("called get_payload_v4 with {}", fork_name)), - ), + ForkName::Base + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => Err(Error::UnsupportedForkVariant(format!( + "called get_payload_v4 with {}", + fork_name + ))), } } @@ -1235,11 +1229,11 @@ impl HttpJsonRpc { } } NewPayloadRequest::Electra(new_payload_request_electra) => { - if engine_capabilities.new_payload_v3 { + if engine_capabilities.new_payload_v4 { self.new_payload_v4_electra(new_payload_request_electra) .await } else { - Err(Error::RequiredMethodUnsupported("engine_newPayloadV3")) + Err(Error::RequiredMethodUnsupported("engine_newPayloadV4")) } } } @@ -1263,13 +1257,18 @@ impl HttpJsonRpc { Err(Error::RequiredMethodUnsupported("engine_getPayload")) } } - ForkName::Deneb | ForkName::Electra => { + ForkName::Deneb => { + if engine_capabilities.get_payload_v3 { + self.get_payload_v3(fork_name, payload_id).await + } else { + Err(Error::RequiredMethodUnsupported("engine_getPayloadv3")) + } + } + ForkName::Electra => { if engine_capabilities.get_payload_v4 { self.get_payload_v4(fork_name, payload_id).await - } else if engine_capabilities.get_payload_v3 { - self.get_payload_v3(fork_name, payload_id).await } else { - Err(Error::RequiredMethodUnsupported("engine_getPayloadV3")) + Err(Error::RequiredMethodUnsupported("engine_getPayloadv4")) } } ForkName::Base | ForkName::Altair => Err(Error::UnsupportedForkVariant(format!( From 936fe60dbc99ae07d5119deaa437fe506941a6df Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 7 May 2024 16:13:36 -0700 Subject: [PATCH 020/227] Handle new engine api methods in mock EL --- .../src/test_utils/handle_rpc.rs | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 1dc8f0ab83e..5e31646d31a 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -111,20 +111,11 @@ pub async fn handle_rpc( .map(|jep| JsonExecutionPayload::V1(jep)) }) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, - ENGINE_NEW_PAYLOAD_V3 => get_param::>(params, 0) + ENGINE_NEW_PAYLOAD_V3 => get_param::>(params, 0) + .map(|jep| JsonExecutionPayload::V3(jep)) + .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, + ENGINE_NEW_PAYLOAD_V4 => get_param::>(params, 0) .map(|jep| JsonExecutionPayload::V4(jep)) - .or_else(|_| { - get_param::>(params, 0) - .map(|jep| JsonExecutionPayload::V3(jep)) - .or_else(|_| { - get_param::>(params, 0) - .map(|jep| JsonExecutionPayload::V2(jep)) - .or_else(|_| { - get_param::>(params, 0) - .map(|jep| JsonExecutionPayload::V1(jep)) - }) - }) - }) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, _ => unreachable!(), }; @@ -190,7 +181,10 @@ pub async fn handle_rpc( } } ForkName::Electra => { - if method == ENGINE_NEW_PAYLOAD_V1 || method == ENGINE_NEW_PAYLOAD_V2 { + if method == ENGINE_NEW_PAYLOAD_V1 + || method == ENGINE_NEW_PAYLOAD_V2 + || method == ENGINE_NEW_PAYLOAD_V3 + { return Err(( format!("{} called after Electra fork!", method), GENERIC_ERROR_CODE, @@ -259,7 +253,10 @@ pub async fn handle_rpc( Ok(serde_json::to_value(JsonPayloadStatusV1::from(response)).unwrap()) } - ENGINE_GET_PAYLOAD_V1 | ENGINE_GET_PAYLOAD_V2 | ENGINE_GET_PAYLOAD_V3 => { + ENGINE_GET_PAYLOAD_V1 + | ENGINE_GET_PAYLOAD_V2 + | ENGINE_GET_PAYLOAD_V3 + | ENGINE_GET_PAYLOAD_V4 => { let request: JsonPayloadIdRequest = get_param(params, 0).map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?; let id = request.into(); @@ -309,7 +306,9 @@ pub async fn handle_rpc( .read() .get_fork_at_timestamp(response.timestamp()) == ForkName::Electra - && method == ENGINE_GET_PAYLOAD_V1 + && (method == ENGINE_GET_PAYLOAD_V1 + || method == ENGINE_GET_PAYLOAD_V2 + || method == ENGINE_GET_PAYLOAD_V3) { return Err(( format!("{} called after Electra fork!", method), @@ -353,6 +352,9 @@ pub async fn handle_rpc( }) .unwrap() } + _ => unreachable!(), + }), + ENGINE_GET_PAYLOAD_V4 => Ok(match JsonExecutionPayload::from(response) { JsonExecutionPayload::V4(execution_payload) => { serde_json::to_value(JsonGetPayloadResponseV4 { execution_payload, From c0d09b4d6cf6bc168002ae686b310d0129958bc6 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 7 May 2024 16:21:21 -0700 Subject: [PATCH 021/227] Note todo --- beacon_node/execution_layer/src/engine_api/json_structures.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 822a3802a85..c9a7c072f8c 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -105,6 +105,7 @@ pub struct JsonExecutionPayload { #[serde(with = "serde_utils::u64_hex_be")] pub excess_blob_gas: u64, #[superstruct(only(V4))] + // TODO(electra): Field name should be changed post devnet-0. See https://github.com/ethereum/execution-apis/pull/544 pub deposit_requests: VariableList, #[superstruct(only(V4))] pub withdrawal_requests: From 90179d4a88fdabb8b78aea134ee0f46c09040c60 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 8 May 2024 09:32:44 -0700 Subject: [PATCH 022/227] EIP7549 `get_attestation_indices` (#5657) * get attesting indices electra impl * fmt * get tests to pass * fmt * fix some beacon chain tests * fmt * fix slasher test * fmt got me again * fix more tests * fix tests --- .../src/attestation_verification.rs | 139 +++++++--- beacon_node/beacon_chain/src/beacon_chain.rs | 42 ++- .../beacon_chain/src/early_attester_cache.rs | 46 +++- .../beacon_chain/src/observed_aggregates.rs | 37 +-- beacon_node/beacon_chain/src/test_utils.rs | 62 +++-- .../tests/attestation_verification.rs | 246 +++++++++++++----- .../beacon_chain/tests/block_verification.rs | 18 +- .../tests/payload_invalidation.rs | 2 +- .../http_api/src/block_packing_efficiency.rs | 21 +- beacon_node/http_api/tests/tests.rs | 3 +- beacon_node/operation_pool/src/attestation.rs | 2 +- .../operation_pool/src/attestation_storage.rs | 73 ++++-- beacon_node/store/src/consensus_context.rs | 9 +- consensus/fork_choice/tests/tests.rs | 9 +- .../src/common/get_attesting_indices.rs | 175 +++++++++++-- .../src/common/get_indexed_attestation.rs | 25 -- consensus/state_processing/src/common/mod.rs | 6 +- .../state_processing/src/consensus_context.rs | 70 ++--- .../base/validator_statuses.rs | 2 +- .../state_processing/src/upgrade/altair.rs | 4 +- consensus/types/src/aggregate_and_proof.rs | 12 +- consensus/types/src/attestation.rs | 25 +- consensus/types/src/beacon_block.rs | 14 +- consensus/types/src/beacon_state.rs | 1 + consensus/types/src/eth_spec.rs | 8 +- consensus/types/src/indexed_attestation.rs | 2 +- .../types/src/signed_aggregate_and_proof.rs | 6 +- lcli/src/indexed_attestations.rs | 13 +- slasher/src/attester_record.rs | 10 +- slasher/src/test_utils.rs | 32 ++- testing/ef_tests/src/handler.rs | 8 + testing/ef_tests/src/type_name.rs | 7 +- testing/ef_tests/tests/tests.rs | 38 ++- 33 files changed, 846 insertions(+), 321 deletions(-) delete mode 100644 consensus/state_processing/src/common/get_indexed_attestation.rs diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index dff3fd6f2dd..28091847683 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -40,12 +40,13 @@ use crate::{ BeaconChain, BeaconChainError, BeaconChainTypes, }; use bls::verify_signature_sets; +use itertools::Itertools; use proto_array::Block as ProtoBlock; use slog::debug; use slot_clock::SlotClock; use state_processing::{ - common::get_indexed_attestation, - per_block_processing::errors::AttestationValidationError, + common::{attesting_indices_base, attesting_indices_electra}, + per_block_processing::errors::{AttestationValidationError, BlockOperationError}, signature_sets::{ indexed_attestation_signature_set_from_pubkeys, signed_aggregate_selection_proof_signature_set, signed_aggregate_signature_set, @@ -55,8 +56,9 @@ use std::borrow::Cow; use strum::AsRefStr; use tree_hash::TreeHash; use types::{ - Attestation, AttestationRef, BeaconCommittee, ChainSpec, CommitteeIndex, Epoch, EthSpec, - ForkName, Hash256, IndexedAttestation, SelectionProof, SignedAggregateAndProof, Slot, SubnetId, + Attestation, AttestationRef, BeaconCommittee, BeaconStateError::NoCommitteeFound, ChainSpec, + CommitteeIndex, Epoch, EthSpec, ForkName, Hash256, IndexedAttestation, SelectionProof, + SignedAggregateAndProof, Slot, SubnetId, }; pub use batch::{batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations}; @@ -545,32 +547,59 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { }; let get_indexed_attestation_with_committee = - |(committee, _): (BeaconCommittee, CommitteesPerSlot)| { - // Note: this clones the signature which is known to be a relatively slow operation. - // - // Future optimizations should remove this clone. - let selection_proof = - SelectionProof::from(signed_aggregate.message().selection_proof().clone()); - - if !selection_proof - .is_aggregator(committee.committee.len(), &chain.spec) - .map_err(|e| Error::BeaconChainError(e.into()))? - { - return Err(Error::InvalidSelectionProof { aggregator_index }); - } - - // Ensure the aggregator is a member of the committee for which it is aggregating. - if !committee.committee.contains(&(aggregator_index as usize)) { - return Err(Error::AggregatorNotInCommittee { aggregator_index }); + |(committees, _): (Vec, CommitteesPerSlot)| { + match attestation { + AttestationRef::Base(att) => { + let committee = committees + .iter() + .filter(|&committee| committee.index == att.data.index) + .at_most_one() + .map_err(|_| Error::NoCommitteeForSlotAndIndex { + slot: att.data.slot, + index: att.data.index, + })?; + + if let Some(committee) = committee { + // Note: this clones the signature which is known to be a relatively slow operation. + // + // Future optimizations should remove this clone. + let selection_proof = SelectionProof::from( + signed_aggregate.message().selection_proof().clone(), + ); + + if !selection_proof + .is_aggregator(committee.committee.len(), &chain.spec) + .map_err(|e| Error::BeaconChainError(e.into()))? + { + return Err(Error::InvalidSelectionProof { aggregator_index }); + } + + // Ensure the aggregator is a member of the committee for which it is aggregating. + if !committee.committee.contains(&(aggregator_index as usize)) { + return Err(Error::AggregatorNotInCommittee { aggregator_index }); + } + attesting_indices_base::get_indexed_attestation( + committee.committee, + att, + ) + .map_err(|e| BeaconChainError::from(e).into()) + } else { + Err(Error::NoCommitteeForSlotAndIndex { + slot: att.data.slot, + index: att.data.index, + }) + } + } + AttestationRef::Electra(att) => { + attesting_indices_electra::get_indexed_attestation(&committees, att) + .map_err(|e| BeaconChainError::from(e).into()) + } } - - get_indexed_attestation(committee.committee, attestation) - .map_err(|e| BeaconChainError::from(e).into()) }; - let indexed_attestation = match map_attestation_committee( + let indexed_attestation = match map_attestation_committees( chain, - attestation, + &attestation, get_indexed_attestation_with_committee, ) { Ok(indexed_attestation) => indexed_attestation, @@ -1252,13 +1281,49 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( chain: &BeaconChain, attestation: AttestationRef, ) -> Result<(IndexedAttestation, CommitteesPerSlot), Error> { - map_attestation_committee(chain, attestation, |(committee, committees_per_slot)| { - get_indexed_attestation(committee.committee, attestation) - .map(|attestation| (attestation, committees_per_slot)) - .map_err(Error::Invalid) + map_attestation_committees(chain, &attestation, |(committees, committees_per_slot)| { + match attestation { + AttestationRef::Base(att) => { + let committee = committees + .iter() + .filter(|&committee| committee.index == att.data.index) + .at_most_one() + .map_err(|_| Error::NoCommitteeForSlotAndIndex { + slot: att.data.slot, + index: att.data.index, + })?; + + if let Some(committee) = committee { + attesting_indices_base::get_indexed_attestation(committee.committee, att) + .map(|attestation| (attestation, committees_per_slot)) + .map_err(Error::Invalid) + } else { + Err(Error::NoCommitteeForSlotAndIndex { + slot: att.data.slot, + index: att.data.index, + }) + } + } + AttestationRef::Electra(att) => { + attesting_indices_electra::get_indexed_attestation(&committees, att) + .map(|attestation| (attestation, committees_per_slot)) + .map_err(|e| { + if e == BlockOperationError::BeaconStateError(NoCommitteeFound) { + Error::NoCommitteeForSlotAndIndex { + slot: att.data.slot, + index: att.committee_index(), + } + } else { + Error::Invalid(e) + } + }) + } + } }) } +// TODO(electra) update comments below to reflect logic changes +// i.e. this now runs the map_fn on a list of committees for the slot of the provided attestation /// Runs the `map_fn` with the committee and committee count per slot for the given `attestation`. /// /// This function exists in this odd "map" pattern because efficiently obtaining the committee for @@ -1268,14 +1333,14 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( /// /// If the committee for `attestation` isn't found in the `shuffling_cache`, we will read a state /// from disk and then update the `shuffling_cache`. -fn map_attestation_committee( +fn map_attestation_committees( chain: &BeaconChain, - attestation: AttestationRef, + attestation: &AttestationRef, map_fn: F, ) -> Result where T: BeaconChainTypes, - F: Fn((BeaconCommittee, CommitteesPerSlot)) -> Result, + F: Fn((Vec, CommitteesPerSlot)) -> Result, { let attestation_epoch = attestation.data().slot.epoch(T::EthSpec::slots_per_epoch()); let target = &attestation.data().target; @@ -1301,12 +1366,12 @@ where let committees_per_slot = committee_cache.committees_per_slot(); Ok(committee_cache - .get_beacon_committee(attestation.data().slot, attestation.data().index) - .map(|committee| map_fn((committee, committees_per_slot))) - .unwrap_or_else(|| { + .get_beacon_committees_at_slot(attestation.data().slot) + .map(|committees| map_fn((committees, committees_per_slot))) + .unwrap_or_else(|_| { Err(Error::NoCommitteeForSlotAndIndex { slot: attestation.data().slot, - index: attestation.data().index, + index: attestation.committee_index(), }) })) }) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index cd47068977b..973dcaadb47 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1917,18 +1917,36 @@ impl BeaconChain { }; drop(cache_timer); - // TODO(electra) implement electra variant - Ok(Attestation::Base(AttestationBase { - aggregation_bits: BitList::with_capacity(committee_len)?, - data: AttestationData { - slot: request_slot, - index: request_index, - beacon_block_root, - source: justified_checkpoint, - target, - }, - signature: AggregateSignature::empty(), - })) + if self.spec.fork_name_at_slot::(request_slot) >= ForkName::Electra { + let mut committee_bits = BitVector::default(); + if committee_len > 0 { + committee_bits.set(request_index as usize, true)?; + } + Ok(Attestation::Electra(AttestationElectra { + aggregation_bits: BitList::with_capacity(committee_len)?, + data: AttestationData { + slot: request_slot, + index: 0u64, + beacon_block_root, + source: justified_checkpoint, + target, + }, + committee_bits, + signature: AggregateSignature::empty(), + })) + } else { + Ok(Attestation::Base(AttestationBase { + aggregation_bits: BitList::with_capacity(committee_len)?, + data: AttestationData { + slot: request_slot, + index: request_index, + beacon_block_root, + source: justified_checkpoint, + target, + }, + signature: AggregateSignature::empty(), + })) + } } /// Performs the same validation as `Self::verify_unaggregated_attestation_for_gossip`, but for diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index 595f941235c..8ed4e5db40b 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -124,18 +124,40 @@ impl EarlyAttesterCache { .get_committee_length::(request_slot, request_index, spec)?; // TODO(electra) make fork-agnostic - let attestation = Attestation::Base(AttestationBase { - aggregation_bits: BitList::with_capacity(committee_len) - .map_err(BeaconStateError::from)?, - data: AttestationData { - slot: request_slot, - index: request_index, - beacon_block_root: item.beacon_block_root, - source: item.source, - target: item.target, - }, - signature: AggregateSignature::empty(), - }); + let attestation = if spec.fork_name_at_slot::(request_slot) >= ForkName::Electra { + let mut committee_bits = BitVector::default(); + if committee_len > 0 { + committee_bits + .set(request_index as usize, true) + .map_err(BeaconStateError::from)?; + } + Attestation::Electra(AttestationElectra { + aggregation_bits: BitList::with_capacity(committee_len) + .map_err(BeaconStateError::from)?, + committee_bits, + data: AttestationData { + slot: request_slot, + index: 0u64, + beacon_block_root: item.beacon_block_root, + source: item.source, + target: item.target, + }, + signature: AggregateSignature::empty(), + }) + } else { + Attestation::Base(AttestationBase { + aggregation_bits: BitList::with_capacity(committee_len) + .map_err(BeaconStateError::from)?, + data: AttestationData { + slot: request_slot, + index: request_index, + beacon_block_root: item.beacon_block_root, + source: item.source, + target: item.target, + }, + signature: AggregateSignature::empty(), + }) + }; metrics::inc_counter(&metrics::BEACON_EARLY_ATTESTER_CACHE_HITS); diff --git a/beacon_node/beacon_chain/src/observed_aggregates.rs b/beacon_node/beacon_chain/src/observed_aggregates.rs index 857b0edb348..5e161a998b5 100644 --- a/beacon_node/beacon_chain/src/observed_aggregates.rs +++ b/beacon_node/beacon_chain/src/observed_aggregates.rs @@ -17,11 +17,8 @@ pub type ObservedSyncContributions = ObservedAggregates< E, BitVector<::SyncSubcommitteeSize>, >; -pub type ObservedAggregateAttestations = ObservedAggregates< - Attestation, - E, - BitList<::MaxValidatorsPerCommittee>, ->; +pub type ObservedAggregateAttestations = + ObservedAggregates, E, BitList<::MaxValidatorsPerSlot>>; /// A trait use to associate capacity constants with the type being stored in `ObservedAggregates`. pub trait Consts { @@ -103,29 +100,39 @@ pub trait SubsetItem { } impl<'a, E: EthSpec> SubsetItem for AttestationRef<'a, E> { - type Item = BitList; + type Item = BitList; fn is_subset(&self, other: &Self::Item) -> bool { match self { - Self::Base(att) => att.aggregation_bits.is_subset(other), - // TODO(electra) implement electra variant - Self::Electra(_) => todo!(), + Self::Base(att) => { + if let Ok(extended_aggregation_bits) = att.extend_aggregation_bits() { + return extended_aggregation_bits.is_subset(other); + } + false + } + Self::Electra(att) => att.aggregation_bits.is_subset(other), } } fn is_superset(&self, other: &Self::Item) -> bool { match self { - Self::Base(att) => other.is_subset(&att.aggregation_bits), - // TODO(electra) implement electra variant - Self::Electra(_) => todo!(), + Self::Base(att) => { + if let Ok(extended_aggregation_bits) = att.extend_aggregation_bits() { + return other.is_subset(&extended_aggregation_bits); + } + false + } + Self::Electra(att) => other.is_subset(&att.aggregation_bits), } } /// Returns the sync contribution aggregation bits. fn get_item(&self) -> Self::Item { match self { - Self::Base(att) => att.aggregation_bits.clone(), - // TODO(electra) implement electra variant - Self::Electra(_) => todo!(), + Self::Base(att) => { + // TODO(electra) fix unwrap + att.extend_aggregation_bits().unwrap() + } + Self::Electra(att) => att.aggregation_bits.clone(), } } diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index ff439742d11..edfff4bf81e 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1032,21 +1032,40 @@ where *state.get_block_root(target_slot)? }; - // TODO(electra) make test fork-agnostic - Ok(Attestation::Base(AttestationBase { - aggregation_bits: BitList::with_capacity(committee_len)?, - data: AttestationData { - slot, - index, - beacon_block_root, - source: state.current_justified_checkpoint(), - target: Checkpoint { - epoch, - root: target_root, + if self.spec.fork_name_at_slot::(slot) >= ForkName::Electra { + let mut committee_bits = BitVector::default(); + committee_bits.set(index as usize, true)?; + Ok(Attestation::Electra(AttestationElectra { + aggregation_bits: BitList::with_capacity(committee_len)?, + committee_bits, + data: AttestationData { + slot, + index: 0u64, + beacon_block_root, + source: state.current_justified_checkpoint(), + target: Checkpoint { + epoch, + root: target_root, + }, }, - }, - signature: AggregateSignature::empty(), - })) + signature: AggregateSignature::empty(), + })) + } else { + Ok(Attestation::Base(AttestationBase { + aggregation_bits: BitList::with_capacity(committee_len)?, + data: AttestationData { + slot, + index, + beacon_block_root, + source: state.current_justified_checkpoint(), + target: Checkpoint { + epoch, + root: target_root, + }, + }, + signature: AggregateSignature::empty(), + })) + } } /// A list of attestations for each committee for the given slot. @@ -1121,11 +1140,14 @@ where ) .unwrap(); - attestation - .aggregation_bits_base_mut() - .unwrap() - .set(i, true) - .unwrap(); + match attestation { + Attestation::Base(ref mut att) => { + att.aggregation_bits.set(i, true).unwrap() + } + Attestation::Electra(ref mut att) => { + att.aggregation_bits.set(i, true).unwrap() + } + } *attestation.signature_mut() = { let domain = self.spec.get_domain( @@ -1366,7 +1388,7 @@ where let signed_aggregate = SignedAggregateAndProof::from_aggregate( aggregator_index as u64, - aggregate, + aggregate.to_ref(), None, &self.validator_keypairs[aggregator_index].sk, &fork, diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index 1327348713c..d132b92a5c2 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -14,14 +14,17 @@ use beacon_chain::{ use genesis::{interop_genesis_state, DEFAULT_ETH1_BLOCK_HASH}; use int_to_bytes::int_to_bytes32; use lazy_static::lazy_static; +use ssz_types::BitVector; use state_processing::{ per_block_processing::errors::AttestationValidationError, per_slot_processing, }; use tree_hash::TreeHash; use types::{ + signed_aggregate_and_proof::SignedAggregateAndProofRefMut, test_utils::generate_deterministic_keypair, Address, AggregateSignature, Attestation, - BeaconStateError, BitList, ChainSpec, Epoch, EthSpec, ForkName, Hash256, Keypair, - MainnetEthSpec, SecretKey, SelectionProof, SignedAggregateAndProof, Slot, SubnetId, Unsigned, + AttestationRef, AttestationRefMut, BeaconStateError, BitList, ChainSpec, Epoch, EthSpec, + ForkName, Hash256, Keypair, MainnetEthSpec, SecretKey, SelectionProof, SignedAggregateAndProof, + Slot, SubnetId, Unsigned, }; pub type E = MainnetEthSpec; @@ -198,7 +201,7 @@ fn get_valid_aggregated_attestation( let signed_aggregate = SignedAggregateAndProof::from_aggregate( aggregator_index as u64, - aggregate, + aggregate.to_ref(), None, &aggregator_sk, &state.fork(), @@ -213,12 +216,13 @@ fn get_valid_aggregated_attestation( /// attestation. fn get_non_aggregator( chain: &BeaconChain, - aggregate: &Attestation, + aggregate: &AttestationRef, ) -> (usize, SecretKey) { let head = chain.head_snapshot(); let state = &head.beacon_state; let current_slot = chain.slot().expect("should get slot"); + // TODO(electra) make fork-agnostic let committee = state .get_beacon_committee(current_slot, aggregate.data().index) .expect("should get committees"); @@ -305,11 +309,15 @@ impl GossipTester { let (mut invalid_aggregate, _, _) = get_valid_aggregated_attestation(&harness.chain, invalid_attestation.clone()); - invalid_aggregate.message.aggregator_index = invalid_aggregate - .message - .aggregator_index - .checked_sub(1) - .unwrap(); + + match invalid_aggregate.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.aggregator_index = att.message.aggregator_index.checked_sub(1).unwrap(); + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.aggregator_index = att.message.aggregator_index.checked_sub(1).unwrap(); + } + } Self { harness, @@ -361,7 +369,10 @@ impl GossipTester { } pub fn non_aggregator(&self) -> (usize, SecretKey) { - get_non_aggregator(&self.harness.chain, &self.valid_aggregate.message.aggregate) + get_non_aggregator( + &self.harness.chain, + &self.valid_aggregate.message().aggregate(), + ) } pub fn import_valid_aggregate(self) -> Self { @@ -490,7 +501,14 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "aggregate from future slot", - |tester, a| a.message.aggregate.data_mut().slot = tester.slot() + 1, + |tester, a| match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.aggregate.data.slot = tester.slot() + 1 + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.aggregate.data.slot = tester.slot() + 1 + } + }, |tester, err| { assert!(matches!( err, @@ -504,9 +522,18 @@ async fn aggregated_gossip_verification() { "aggregate from past slot", |tester, a| { let too_early_slot = tester.earliest_valid_attestation_slot() - 1; - a.message.aggregate.data_mut().slot = too_early_slot; - a.message.aggregate.data_mut().target.epoch = - too_early_slot.epoch(E::slots_per_epoch()); + match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.aggregate.data.slot = too_early_slot; + att.message.aggregate.data.target.epoch = + too_early_slot.epoch(E::slots_per_epoch()); + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.aggregate.data.slot = too_early_slot; + att.message.aggregate.data.target.epoch = + too_early_slot.epoch(E::slots_per_epoch()); + } + } }, |tester, err| { let valid_early_slot = tester.earliest_valid_attestation_slot(); @@ -530,7 +557,14 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "attestation with invalid target epoch", - |_, a| a.message.aggregate.data_mut().target.epoch += 1, + |_, a| match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.aggregate.data.target.epoch += 1 + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.aggregate.data.target.epoch += 1 + } + }, |_, err| assert!(matches!(err, AttnError::InvalidTargetEpoch { .. })), ) /* @@ -539,7 +573,14 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "attestation with invalid target root", - |_, a| a.message.aggregate.data_mut().target.root = Hash256::repeat_byte(42), + |_, a| match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.aggregate.data.target.root = Hash256::repeat_byte(42) + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.aggregate.data.target.root = Hash256::repeat_byte(42) + } + }, |_, err| assert!(matches!(err, AttnError::InvalidTargetRoot { .. })), ) /* @@ -549,7 +590,14 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "aggregate with unknown head block", - |_, a| a.message.aggregate.data_mut().beacon_block_root = Hash256::repeat_byte(42), + |_, a| match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.aggregate.data.beacon_block_root = Hash256::repeat_byte(42) + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.aggregate.data.beacon_block_root = Hash256::repeat_byte(42) + } + }, |_, err| { assert!(matches!( err, @@ -567,20 +615,19 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "aggregate with no participants", - |_, a| { - match &mut a.message.aggregate { - Attestation::Base(ref mut att) => { - let aggregation_bits = &mut att.aggregation_bits; - aggregation_bits.difference_inplace(&aggregation_bits.clone()); - assert!(aggregation_bits.is_zero()); - } - Attestation::Electra(ref mut att) => { - let aggregation_bits = &mut att.aggregation_bits; - aggregation_bits.difference_inplace(&aggregation_bits.clone()); - assert!(aggregation_bits.is_zero()); - } + |_, a| match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + let aggregation_bits = &mut att.message.aggregate.aggregation_bits; + aggregation_bits.difference_inplace(&aggregation_bits.clone()); + assert!(aggregation_bits.is_zero()); + att.message.aggregate.signature = AggregateSignature::infinity() + } + SignedAggregateAndProofRefMut::Electra(att) => { + let aggregation_bits = &mut att.message.aggregate.aggregation_bits; + aggregation_bits.difference_inplace(&aggregation_bits.clone()); + assert!(aggregation_bits.is_zero()); + att.message.aggregate.signature = AggregateSignature::infinity() } - *a.message.aggregate.signature_mut() = AggregateSignature::infinity(); }, |_, err| assert!(matches!(err, AttnError::EmptyAggregationBitfield)), ) @@ -591,7 +638,14 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "aggregate with bad signature", - |tester, a| a.signature = tester.aggregator_sk.sign(Hash256::repeat_byte(42)), + |tester, a| match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.signature = tester.aggregator_sk.sign(Hash256::repeat_byte(42)) + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.signature = tester.aggregator_sk.sign(Hash256::repeat_byte(42)) + } + }, |_, err| assert!(matches!(err, AttnError::InvalidSignature)), ) /* @@ -608,7 +662,7 @@ async fn aggregated_gossip_verification() { .chain .head_snapshot() .beacon_state - .get_beacon_committee(tester.slot(), a.message.aggregate.data().index) + .get_beacon_committee(tester.slot(), a.message().aggregate().data().index) .expect("should get committees") .committee .len(); @@ -618,19 +672,38 @@ async fn aggregated_gossip_verification() { // // Could run for ever, but that seems _really_ improbable. let mut i: u64 = 0; - a.message.selection_proof = loop { - i += 1; - let proof: SelectionProof = tester - .aggregator_sk - .sign(Hash256::from_slice(&int_to_bytes32(i))) - .into(); - if proof - .is_aggregator(committee_len, &tester.harness.chain.spec) - .unwrap() - { - break proof.into(); + match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.selection_proof = loop { + i += 1; + let proof: SelectionProof = tester + .aggregator_sk + .sign(Hash256::from_slice(&int_to_bytes32(i))) + .into(); + if proof + .is_aggregator(committee_len, &tester.harness.chain.spec) + .unwrap() + { + break proof.into(); + } + }; + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.selection_proof = loop { + i += 1; + let proof: SelectionProof = tester + .aggregator_sk + .sign(Hash256::from_slice(&int_to_bytes32(i))) + .into(); + if proof + .is_aggregator(committee_len, &tester.harness.chain.spec) + .unwrap() + { + break proof.into(); + } + }; } - }; + } }, |_, err| assert!(matches!(err, AttnError::InvalidSignature)), ) @@ -644,7 +717,14 @@ async fn aggregated_gossip_verification() { |tester, a| { let mut agg_sig = AggregateSignature::infinity(); agg_sig.add_assign(&tester.aggregator_sk.sign(Hash256::repeat_byte(42))); - *a.message.aggregate.signature_mut() = agg_sig; + match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.aggregate.signature = agg_sig; + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.aggregate.signature = agg_sig; + } + } }, |_, err| assert!(matches!(err, AttnError::InvalidSignature)), ) @@ -653,8 +733,15 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "aggregate with too-high aggregator index", - |_, a| { - a.message.aggregator_index = ::ValidatorRegistryLimit::to_u64() + 1 + |_, a| match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.aggregator_index = + ::ValidatorRegistryLimit::to_u64() + 1 + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.aggregator_index = + ::ValidatorRegistryLimit::to_u64() + 1 + } }, |_, err| { assert!(matches!( @@ -673,7 +760,14 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "aggregate with unknown aggregator index", - |_, a| a.message.aggregator_index = VALIDATOR_COUNT as u64, + |_, a| match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.aggregator_index = VALIDATOR_COUNT as u64 + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.aggregator_index = VALIDATOR_COUNT as u64 + } + }, |_, err| { assert!(matches!( err, @@ -681,10 +775,15 @@ async fn aggregated_gossip_verification() { // // AttnError::AggregatorPubkeyUnknown(unknown_validator) // - // However the following error is triggered first: + // However, the following error is triggered first: AttnError::AggregatorNotInCommittee { aggregator_index - } + } | + // unless were working with electra attestations + // in which case this error is triggered instead: + AttnError::AggregatorPubkeyUnknown( + aggregator_index + ) if aggregator_index == VALIDATOR_COUNT as u64 )) }, @@ -703,7 +802,7 @@ async fn aggregated_gossip_verification() { let (index, sk) = tester.non_aggregator(); *a = SignedAggregateAndProof::from_aggregate( index as u64, - tester.valid_aggregate.message.aggregate.clone(), + tester.valid_aggregate.message().aggregate().clone(), None, &sk, &chain.canonical_head.cached_head().head_fork(), @@ -739,7 +838,7 @@ async fn aggregated_gossip_verification() { assert!(matches!( err, AttnError::AttestationSupersetKnown(hash) - if hash == tester.valid_aggregate.message.aggregate.data().tree_hash_root() + if hash == tester.valid_aggregate.message().aggregate().data().tree_hash_root() )) }, ) @@ -751,7 +850,14 @@ async fn aggregated_gossip_verification() { */ .inspect_aggregate_err( "aggregate from aggregator that has already been seen", - |_, a| a.message.aggregate.data_mut().beacon_block_root = Hash256::repeat_byte(42), + |_, a| match a.to_mut() { + SignedAggregateAndProofRefMut::Base(att) => { + att.message.aggregate.data.beacon_block_root = Hash256::repeat_byte(42) + } + SignedAggregateAndProofRefMut::Electra(att) => { + att.message.aggregate.data.beacon_block_root = Hash256::repeat_byte(42) + } + }, |tester, err| { assert!(matches!( err, @@ -776,13 +882,29 @@ async fn unaggregated_gossip_verification() { .inspect_unaggregate_err( "attestation with invalid committee index", |tester, a, _| { - a.data_mut().index = tester - .harness - .chain - .head_snapshot() - .beacon_state - .get_committee_count_at_slot(a.data().slot) - .unwrap() + match a.to_mut() { + AttestationRefMut::Base(attn) => { + attn.data.index = tester + .harness + .chain + .head_snapshot() + .beacon_state + .get_committee_count_at_slot(attn.data.slot) + .unwrap(); + } + AttestationRefMut::Electra(attn) => { + let committee_index = tester + .harness + .chain + .head_snapshot() + .beacon_state + .get_committee_count_at_slot(attn.data.slot) + .unwrap(); + // overwrite the existing committee bits before setting + attn.committee_bits = BitVector::default(); + attn.committee_bits.set(committee_index as usize, true).unwrap(); + } + } }, |_, err| assert!(matches!(err, AttnError::NoCommitteeForSlotAndIndex { .. })), ) @@ -1325,8 +1447,8 @@ async fn verify_aggregate_for_gossip_doppelganger_detection() { .verify_aggregated_attestation_for_gossip(&valid_aggregate) .expect("should verify aggregate attestation"); - let epoch = valid_aggregate.message.aggregate.data().target.epoch; - let index = valid_aggregate.message.aggregator_index as usize; + let epoch = valid_aggregate.message().aggregate().data().target.epoch; + let index = valid_aggregate.message().aggregator_index() as usize; assert!(harness.chain.validator_seen_at_epoch(index, epoch)); // Check the correct beacon cache is populated diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index e24894b242c..257b6d53463 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -13,7 +13,7 @@ use lazy_static::lazy_static; use logging::test_logger; use slasher::{Config as SlasherConfig, Slasher}; use state_processing::{ - common::get_indexed_attestation, + common::{attesting_indices_base, attesting_indices_electra}, per_block_processing::{per_block_processing, BlockSignatureStrategy}, per_slot_processing, BlockProcessingError, ConsensusContext, VerifyBlockRoot, }; @@ -1258,11 +1258,17 @@ async fn verify_block_for_gossip_doppelganger_detection() { for att in attestations.iter() { let epoch = att.data().target.epoch; - let committee = state - .get_beacon_committee(att.data().slot, att.data().index) - .unwrap(); - let indexed_attestation = - get_indexed_attestation(committee.committee, att.to_ref()).unwrap(); + let indexed_attestation = match att { + Attestation::Base(att) => { + let committee = state + .get_beacon_committee(att.data.slot, att.data.index) + .unwrap(); + attesting_indices_base::get_indexed_attestation(committee.committee, att).unwrap() + } + Attestation::Electra(att) => { + attesting_indices_electra::get_indexed_attestation_from_state(&state, att).unwrap() + } + }; match indexed_attestation { IndexedAttestation::Base(indexed_attestation) => { diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index c1c770cfe4e..8c9957db169 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -1207,7 +1207,7 @@ async fn attesting_to_optimistic_head() { .chain .naive_aggregation_pool .write() - .insert(&attestation) + .insert(attestation.to_ref()) .unwrap(); attestation diff --git a/beacon_node/http_api/src/block_packing_efficiency.rs b/beacon_node/http_api/src/block_packing_efficiency.rs index 43dee5b6eb4..66c71872786 100644 --- a/beacon_node/http_api/src/block_packing_efficiency.rs +++ b/beacon_node/http_api/src/block_packing_efficiency.rs @@ -132,9 +132,24 @@ impl PackingEfficiencyHandler { } } } - // TODO(electra) implement electra variant - AttestationRef::Electra(_) => { - todo!() + AttestationRef::Electra(attn) => { + for (position, voted) in attn.aggregation_bits.iter().enumerate() { + if voted { + let unique_attestation = UniqueAttestation { + slot: attn.data.slot, + committee_index: attn.data.index, + committee_position: position, + }; + let inclusion_distance: u64 = block + .slot() + .as_u64() + .checked_sub(attn.data.slot.as_u64()) + .ok_or(PackingEfficiencyError::InvalidAttestationError)?; + + self.available_attestations.remove(&unique_attestation); + attestations_in_block.insert(unique_attestation, inclusion_distance); + } + } } } } diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 3653929bdab..315df8fa9c5 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -3285,6 +3285,7 @@ impl ApiTester { .unwrap() .data; + // TODO(electra) make fork-agnostic let mut attestation = Attestation::Base(AttestationBase { aggregation_bits: BitList::with_capacity(duty.committee_length as usize).unwrap(), data: attestation_data, @@ -3303,7 +3304,7 @@ impl ApiTester { SignedAggregateAndProof::from_aggregate( i as u64, - attestation, + attestation.to_ref(), Some(proof), &kp.sk, &fork, diff --git a/beacon_node/operation_pool/src/attestation.rs b/beacon_node/operation_pool/src/attestation.rs index 60074cde740..207e2c65e45 100644 --- a/beacon_node/operation_pool/src/attestation.rs +++ b/beacon_node/operation_pool/src/attestation.rs @@ -2,7 +2,7 @@ use crate::attestation_storage::{AttestationRef, CompactIndexedAttestation}; use crate::max_cover::MaxCover; use crate::reward_cache::RewardCache; use state_processing::common::{ - base, get_attestation_participation_flag_indices, get_attesting_indices, + attesting_indices_base::get_attesting_indices, base, get_attestation_participation_flag_indices, }; use std::collections::HashMap; use types::{ diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index 927e8c9b12c..7d7a72704d9 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -28,7 +28,7 @@ pub struct CompactIndexedAttestation { #[superstruct(only(Base), partial_getter(rename = "aggregation_bits_base"))] pub aggregation_bits: BitList, #[superstruct(only(Electra), partial_getter(rename = "aggregation_bits_electra"))] - pub aggregation_bits: BitList, + pub aggregation_bits: BitList, pub signature: AggregateSignature, pub index: u64, #[superstruct(only(Electra))] @@ -81,8 +81,15 @@ impl SplitAttestation { index: data.index, }) } - // TODO(electra) implement electra variant - Attestation::Electra(_) => todo!(), + Attestation::Electra(attn) => { + CompactIndexedAttestation::Electra(CompactIndexedAttestationElectra { + attesting_indices, + aggregation_bits: attn.aggregation_bits, + signature: attestation.signature().clone(), + index: data.index, + committee_bits: attn.committee_bits, + }) + } }; Self { @@ -157,9 +164,10 @@ impl CompactIndexedAttestation { (CompactIndexedAttestation::Base(this), CompactIndexedAttestation::Base(other)) => { this.signers_disjoint_from(other) } - (CompactIndexedAttestation::Electra(_), CompactIndexedAttestation::Electra(_)) => { - todo!() - } + ( + CompactIndexedAttestation::Electra(this), + CompactIndexedAttestation::Electra(other), + ) => this.signers_disjoint_from(other), // TODO(electra) is a mix of electra and base compact indexed attestations an edge case we need to deal with? _ => false, } @@ -170,9 +178,10 @@ impl CompactIndexedAttestation { (CompactIndexedAttestation::Base(this), CompactIndexedAttestation::Base(other)) => { this.aggregate(other) } - (CompactIndexedAttestation::Electra(_), CompactIndexedAttestation::Electra(_)) => { - todo!() - } + ( + CompactIndexedAttestation::Electra(this), + CompactIndexedAttestation::Electra(other), + ) => this.aggregate(other), // TODO(electra) is a mix of electra and base compact indexed attestations an edge case we need to deal with? _ => (), } @@ -198,13 +207,34 @@ impl CompactIndexedAttestationBase { } } +impl CompactIndexedAttestationElectra { + // TODO(electra) update to match spec requirements + pub fn signers_disjoint_from(&self, other: &Self) -> bool { + self.aggregation_bits + .intersection(&other.aggregation_bits) + .is_zero() + } + + // TODO(electra) update to match spec requirements + pub fn aggregate(&mut self, other: &Self) { + self.attesting_indices = self + .attesting_indices + .drain(..) + .merge(other.attesting_indices.iter().copied()) + .dedup() + .collect(); + self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); + self.signature.add_assign_aggregate(&other.signature); + } +} + impl AttestationMap { pub fn insert(&mut self, attestation: Attestation, attesting_indices: Vec) { let SplitAttestation { checkpoint, data, indexed, - } = SplitAttestation::new(attestation, attesting_indices); + } = SplitAttestation::new(attestation.clone(), attesting_indices); let attestation_map = self.checkpoint_map.entry(checkpoint).or_default(); let attestations = attestation_map.attestations.entry(data).or_default(); @@ -213,14 +243,23 @@ impl AttestationMap { // NOTE: this is sub-optimal and in future we will remove this in favour of max-clique // aggregation. let mut aggregated = false; - for existing_attestation in attestations.iter_mut() { - if existing_attestation.signers_disjoint_from(&indexed) { - existing_attestation.aggregate(&indexed); - aggregated = true; - } else if *existing_attestation == indexed { - aggregated = true; + + match attestation { + Attestation::Base(_) => { + for existing_attestation in attestations.iter_mut() { + if existing_attestation.signers_disjoint_from(&indexed) { + existing_attestation.aggregate(&indexed); + aggregated = true; + } else if *existing_attestation == indexed { + aggregated = true; + } + } } - } + // TODO(electra) in order to be devnet ready, we can skip + // aggregating here for now. this will result in "poorly" + // constructed blocks, but that should be fine for devnet + Attestation::Electra(_) => (), + }; if !aggregated { attestations.push(indexed); diff --git a/beacon_node/store/src/consensus_context.rs b/beacon_node/store/src/consensus_context.rs index 727423d172f..409b364992d 100644 --- a/beacon_node/store/src/consensus_context.rs +++ b/beacon_node/store/src/consensus_context.rs @@ -21,13 +21,8 @@ pub struct OnDiskConsensusContext { /// /// They are not part of the on-disk format. #[ssz(skip_serializing, skip_deserializing)] - indexed_attestations: HashMap< - ( - AttestationData, - BitList, - ), - IndexedAttestation, - >, + indexed_attestations: + HashMap<(AttestationData, BitList), IndexedAttestation>, } impl OnDiskConsensusContext { diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index 63bf3b51bd0..3b05cfab1fd 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -830,8 +830,13 @@ async fn invalid_attestation_empty_bitfield() { .await .apply_attestation_to_chain( MutationDelay::NoDelay, - |attestation, _| { - *attestation.attesting_indices_base_mut().unwrap() = vec![].into(); + |attestation, _| match attestation { + IndexedAttestation::Base(ref mut att) => { + att.attesting_indices = vec![].into(); + } + IndexedAttestation::Electra(ref mut att) => { + att.attesting_indices = vec![].into(); + } }, |result| { assert_invalid_attestation!(result, InvalidAttestation::EmptyAggregationBitfield) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index e798f55f844..595cc69f87c 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -1,25 +1,162 @@ use types::*; -/// Returns validator indices which participated in the attestation, sorted by increasing index. -pub fn get_attesting_indices( - committee: &[usize], - bitlist: &BitList, -) -> Result, BeaconStateError> { - if bitlist.len() != committee.len() { - return Err(BeaconStateError::InvalidBitfield); +pub mod attesting_indices_base { + use crate::per_block_processing::errors::{AttestationInvalid as Invalid, BlockOperationError}; + use types::*; + + /// Convert `attestation` to (almost) indexed-verifiable form. + /// + /// Spec v0.12.1 + pub fn get_indexed_attestation( + committee: &[usize], + attestation: &AttestationBase, + ) -> Result, BlockOperationError> { + let attesting_indices = + get_attesting_indices::(committee, &attestation.aggregation_bits)?; + + Ok(IndexedAttestation::Base(IndexedAttestationBase { + attesting_indices: VariableList::new(attesting_indices)?, + data: attestation.data.clone(), + signature: attestation.signature.clone(), + })) } - let mut indices = Vec::with_capacity(bitlist.num_set_bits()); + /// Returns validator indices which participated in the attestation, sorted by increasing index. + pub fn get_attesting_indices( + committee: &[usize], + bitlist: &BitList, + ) -> Result, BeaconStateError> { + if bitlist.len() != committee.len() { + return Err(BeaconStateError::InvalidBitfield); + } + + let mut indices = Vec::with_capacity(bitlist.num_set_bits()); - for (i, validator_index) in committee.iter().enumerate() { - if let Ok(true) = bitlist.get(i) { - indices.push(*validator_index as u64) + for (i, validator_index) in committee.iter().enumerate() { + if let Ok(true) = bitlist.get(i) { + indices.push(*validator_index as u64) + } } + + indices.sort_unstable(); + + Ok(indices) + } +} + +pub mod attesting_indices_electra { + use std::collections::{HashMap, HashSet}; + + use crate::per_block_processing::errors::{AttestationInvalid as Invalid, BlockOperationError}; + use itertools::Itertools; + use safe_arith::SafeArith; + use types::*; + + pub fn get_indexed_attestation( + committees: &[BeaconCommittee], + attestation: &AttestationElectra, + ) -> Result, BlockOperationError> { + let attesting_indices = get_attesting_indices::( + committees, + &attestation.aggregation_bits, + &attestation.committee_bits, + )?; + + Ok(IndexedAttestation::Electra(IndexedAttestationElectra { + attesting_indices: VariableList::new(attesting_indices)?, + data: attestation.data.clone(), + signature: attestation.signature.clone(), + })) } - indices.sort_unstable(); + pub fn get_indexed_attestation_from_state( + beacon_state: &BeaconState, + attestation: &AttestationElectra, + ) -> Result, BlockOperationError> { + let committees = beacon_state.get_beacon_committees_at_slot(attestation.data.slot)?; + let attesting_indices = get_attesting_indices::( + &committees, + &attestation.aggregation_bits, + &attestation.committee_bits, + )?; + + Ok(IndexedAttestation::Electra(IndexedAttestationElectra { + attesting_indices: VariableList::new(attesting_indices)?, + data: attestation.data.clone(), + signature: attestation.signature.clone(), + })) + } - Ok(indices) + /// Shortcut for getting the attesting indices while fetching the committee from the state's cache. + pub fn get_attesting_indices_from_state( + state: &BeaconState, + att: &AttestationElectra, + ) -> Result, BeaconStateError> { + let committees = state.get_beacon_committees_at_slot(att.data.slot)?; + get_attesting_indices::(&committees, &att.aggregation_bits, &att.committee_bits) + } + + /// Returns validator indices which participated in the attestation, sorted by increasing index. + pub fn get_attesting_indices( + committees: &[BeaconCommittee], + aggregation_bits: &BitList, + committee_bits: &BitVector, + ) -> Result, BeaconStateError> { + let mut output: HashSet = HashSet::new(); + + let committee_indices = get_committee_indices::(committee_bits); + + let committee_offset = 0; + + let committees_map: HashMap = committees + .iter() + .map(|committee| (committee.index, committee)) + .collect(); + + for index in committee_indices { + if let Some(&beacon_committee) = committees_map.get(&index) { + if aggregation_bits.len() != beacon_committee.committee.len() { + return Err(BeaconStateError::InvalidBitfield); + } + let committee_attesters = beacon_committee + .committee + .iter() + .enumerate() + .filter_map(|(i, &index)| { + if let Ok(aggregation_bit_index) = committee_offset.safe_add(i) { + if aggregation_bits.get(aggregation_bit_index).unwrap_or(false) { + return Some(index as u64); + } + } + None + }) + .collect::>(); + + output.extend(committee_attesters); + + committee_offset.safe_add(beacon_committee.committee.len())?; + } else { + return Err(Error::NoCommitteeFound); + } + + // TODO(electra) what should we do when theres no committee found for a given index? + } + + let mut indices = output.into_iter().collect_vec(); + indices.sort_unstable(); + + Ok(indices) + } + + fn get_committee_indices( + committee_bits: &BitVector, + ) -> Vec { + committee_bits + .iter() + .enumerate() + .filter_map(|(index, bit)| if bit { Some(index as u64) } else { None }) + .collect() + } } /// Shortcut for getting the attesting indices while fetching the committee from the state's cache. @@ -27,12 +164,16 @@ pub fn get_attesting_indices_from_state( state: &BeaconState, att: AttestationRef, ) -> Result, BeaconStateError> { - let committee = state.get_beacon_committee(att.data().slot, att.data().index)?; match att { AttestationRef::Base(att) => { - get_attesting_indices::(committee.committee, &att.aggregation_bits) + let committee = state.get_beacon_committee(att.data.slot, att.data.index)?; + attesting_indices_base::get_attesting_indices::( + committee.committee, + &att.aggregation_bits, + ) + } + AttestationRef::Electra(att) => { + attesting_indices_electra::get_attesting_indices_from_state::(state, att) } - // TODO(electra) implement get_attesting_indices for electra - AttestationRef::Electra(_) => todo!(), } } diff --git a/consensus/state_processing/src/common/get_indexed_attestation.rs b/consensus/state_processing/src/common/get_indexed_attestation.rs deleted file mode 100644 index a04d9198c68..00000000000 --- a/consensus/state_processing/src/common/get_indexed_attestation.rs +++ /dev/null @@ -1,25 +0,0 @@ -use super::get_attesting_indices; -use crate::per_block_processing::errors::{AttestationInvalid as Invalid, BlockOperationError}; -use types::{indexed_attestation::IndexedAttestationBase, *}; - -type Result = std::result::Result>; - -/// Convert `attestation` to (almost) indexed-verifiable form. -/// -/// Spec v0.12.1 -pub fn get_indexed_attestation( - committee: &[usize], - attestation: AttestationRef, -) -> Result> { - let attesting_indices = match attestation { - AttestationRef::Base(att) => get_attesting_indices::(committee, &att.aggregation_bits)?, - // TODO(electra) implement get_attesting_indices for electra - AttestationRef::Electra(_) => todo!(), - }; - - Ok(IndexedAttestation::Base(IndexedAttestationBase { - attesting_indices: VariableList::new(attesting_indices)?, - data: attestation.data().clone(), - signature: attestation.signature().clone(), - })) -} diff --git a/consensus/state_processing/src/common/mod.rs b/consensus/state_processing/src/common/mod.rs index cefc47b0235..0287748fd04 100644 --- a/consensus/state_processing/src/common/mod.rs +++ b/consensus/state_processing/src/common/mod.rs @@ -1,7 +1,6 @@ mod deposit_data_tree; mod get_attestation_participation; mod get_attesting_indices; -mod get_indexed_attestation; mod initiate_validator_exit; mod slash_validator; @@ -11,8 +10,9 @@ pub mod update_progressive_balances_cache; pub use deposit_data_tree::DepositDataTree; pub use get_attestation_participation::get_attestation_participation_flag_indices; -pub use get_attesting_indices::{get_attesting_indices, get_attesting_indices_from_state}; -pub use get_indexed_attestation::get_indexed_attestation; +pub use get_attesting_indices::{ + attesting_indices_base, attesting_indices_electra, get_attesting_indices_from_state, +}; pub use initiate_validator_exit::initiate_validator_exit; pub use slash_validator::slash_validator; diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index b28f218fc8a..6b0f2eb8fee 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -1,4 +1,4 @@ -use crate::common::get_indexed_attestation; +use crate::common::{attesting_indices_base, attesting_indices_electra}; use crate::per_block_processing::errors::{AttestationInvalid, BlockOperationError}; use crate::EpochCacheError; use std::collections::{hash_map::Entry, HashMap}; @@ -22,13 +22,8 @@ pub struct ConsensusContext { /// Block root of the block at `slot`. pub current_block_root: Option, /// Cache of indexed attestations constructed during block processing. - pub indexed_attestations: HashMap< - ( - AttestationData, - BitList, - ), - IndexedAttestation, - >, + pub indexed_attestations: + HashMap<(AttestationData, BitList), IndexedAttestation>, } #[derive(Debug, PartialEq, Clone)] @@ -159,32 +154,40 @@ impl ConsensusContext { state: &BeaconState, attestation: AttestationRef<'a, E>, ) -> Result, BlockOperationError> { - let aggregation_bits = match attestation { + match attestation { AttestationRef::Base(attn) => { - let mut extended_aggregation_bits: BitList = - BitList::with_capacity(attn.aggregation_bits.len()) - .map_err(BeaconStateError::from)?; - - for (i, bit) in attn.aggregation_bits.iter().enumerate() { - extended_aggregation_bits - .set(i, bit) - .map_err(BeaconStateError::from)?; + let extended_aggregation_bits = attn + .extend_aggregation_bits() + .map_err(BeaconStateError::from)?; + + let key = (attn.data.clone(), extended_aggregation_bits); + + match self.indexed_attestations.entry(key) { + Entry::Occupied(occupied) => Ok(occupied.into_mut()), + Entry::Vacant(vacant) => { + let committee = + state.get_beacon_committee(attn.data.slot, attn.data.index)?; + let indexed_attestation = attesting_indices_base::get_indexed_attestation( + committee.committee, + attn, + )?; + Ok(vacant.insert(indexed_attestation)) + } } - extended_aggregation_bits } - AttestationRef::Electra(attn) => attn.aggregation_bits.clone(), - }; - - let key = (attestation.data().clone(), aggregation_bits); - - match self.indexed_attestations.entry(key) { - Entry::Occupied(occupied) => Ok(occupied.into_mut()), - Entry::Vacant(vacant) => { - let committee = state - .get_beacon_committee(attestation.data().slot, attestation.data().index)?; - let indexed_attestation = - get_indexed_attestation(committee.committee, attestation)?; - Ok(vacant.insert(indexed_attestation)) + AttestationRef::Electra(attn) => { + let key = (attn.data.clone(), attn.aggregation_bits.clone()); + + match self.indexed_attestations.entry(key) { + Entry::Occupied(occupied) => Ok(occupied.into_mut()), + Entry::Vacant(vacant) => { + let indexed_attestation = + attesting_indices_electra::get_indexed_attestation_from_state( + state, attn, + )?; + Ok(vacant.insert(indexed_attestation)) + } + } } } .map(|indexed_attestation| (*indexed_attestation).to_ref()) @@ -198,10 +201,7 @@ impl ConsensusContext { pub fn set_indexed_attestations( mut self, attestations: HashMap< - ( - AttestationData, - BitList, - ), + (AttestationData, BitList), IndexedAttestation, >, ) -> Self { diff --git a/consensus/state_processing/src/per_epoch_processing/base/validator_statuses.rs b/consensus/state_processing/src/per_epoch_processing/base/validator_statuses.rs index 7e244058038..edf75e7c5eb 100644 --- a/consensus/state_processing/src/per_epoch_processing/base/validator_statuses.rs +++ b/consensus/state_processing/src/per_epoch_processing/base/validator_statuses.rs @@ -1,4 +1,4 @@ -use crate::common::get_attesting_indices; +use crate::common::attesting_indices_base::get_attesting_indices; use safe_arith::SafeArith; use types::{BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, PendingAttestation}; diff --git a/consensus/state_processing/src/upgrade/altair.rs b/consensus/state_processing/src/upgrade/altair.rs index 872560db3df..3006da25ae7 100644 --- a/consensus/state_processing/src/upgrade/altair.rs +++ b/consensus/state_processing/src/upgrade/altair.rs @@ -1,5 +1,7 @@ use crate::common::update_progressive_balances_cache::initialize_progressive_balances_cache; -use crate::common::{get_attestation_participation_flag_indices, get_attesting_indices}; +use crate::common::{ + attesting_indices_base::get_attesting_indices, get_attestation_participation_flag_indices, +}; use std::mem; use std::sync::Arc; use types::{ diff --git a/consensus/types/src/aggregate_and_proof.rs b/consensus/types/src/aggregate_and_proof.rs index b0dbdadb952..e8fa01c143e 100644 --- a/consensus/types/src/aggregate_and_proof.rs +++ b/consensus/types/src/aggregate_and_proof.rs @@ -1,4 +1,4 @@ -use super::{Attestation, AttestationBase, AttestationElectra, AttestationRef}; +use super::{AttestationBase, AttestationElectra, AttestationRef}; use super::{ ChainSpec, Domain, EthSpec, Fork, Hash256, PublicKey, SecretKey, SelectionProof, Signature, SignedRoot, @@ -81,7 +81,7 @@ impl AggregateAndProof { /// If `selection_proof.is_none()` it will be computed locally. pub fn from_aggregate( aggregator_index: u64, - aggregate: Attestation, + aggregate: AttestationRef<'_, E>, selection_proof: Option, secret_key: &SecretKey, fork: &Fork, @@ -101,14 +101,14 @@ impl AggregateAndProof { .into(); match aggregate { - Attestation::Base(attestation) => Self::Base(AggregateAndProofBase { + AttestationRef::Base(attestation) => Self::Base(AggregateAndProofBase { aggregator_index, - aggregate: attestation, + aggregate: attestation.clone(), selection_proof, }), - Attestation::Electra(attestation) => Self::Electra(AggregateAndProofElectra { + AttestationRef::Electra(attestation) => Self::Electra(AggregateAndProofElectra { aggregator_index, - aggregate: attestation, + aggregate: attestation.clone(), selection_proof, }), } diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 4676a100be6..61e63e11e22 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -65,7 +65,7 @@ pub struct Attestation { #[superstruct(only(Base), partial_getter(rename = "aggregation_bits_base"))] pub aggregation_bits: BitList, #[superstruct(only(Electra), partial_getter(rename = "aggregation_bits_electra"))] - pub aggregation_bits: BitList, + pub aggregation_bits: BitList, pub data: AttestationData, pub signature: AggregateSignature, #[superstruct(only(Electra))] @@ -236,6 +236,13 @@ impl<'a, E: EthSpec> AttestationRef<'a, E> { Self::Electra(att) => att.aggregation_bits.num_set_bits(), } } + + pub fn committee_index(&self) -> u64 { + match self { + AttestationRef::Base(att) => att.data.index, + AttestationRef::Electra(att) => att.committee_index(), + } + } } impl AttestationElectra { @@ -330,7 +337,6 @@ impl AttestationBase { pub fn aggregate(&mut self, other: &Self) { debug_assert_eq!(self.data, other.data); debug_assert!(self.signers_disjoint_from(other)); - self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); self.signature.add_assign_aggregate(&other.signature); } @@ -381,6 +387,18 @@ impl AttestationBase { Ok(()) } } + + pub fn extend_aggregation_bits( + &self, + ) -> Result, ssz_types::Error> { + let mut extended_aggregation_bits: BitList = + BitList::with_capacity(self.aggregation_bits.len())?; + + for (i, bit) in self.aggregation_bits.iter().enumerate() { + extended_aggregation_bits.set(i, bit)?; + } + Ok(extended_aggregation_bits) + } } impl SlotData for Attestation { @@ -419,6 +437,9 @@ mod tests { assert_eq!(signature, 288 + 16); let attestation_expected = aggregation_bits + attestation_data + signature; + // TODO(electra) since we've removed attestation aggregation for electra variant + // i've updated the attestation value expected from 488 544 + // assert_eq!(attestation_expected, 488); assert_eq!(attestation_expected, 488); assert_eq!( size_of::>(), diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index ac0525a3d84..09f3306c73d 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -608,12 +608,8 @@ impl> BeaconBlockElectra let base_block: BeaconBlockBase<_, Payload> = BeaconBlockBase::full(spec); // TODO(electra): check this let indexed_attestation: IndexedAttestationElectra = IndexedAttestationElectra { - attesting_indices: VariableList::new(vec![ - 0_u64; - E::MaxValidatorsPerCommitteePerSlot::to_usize( - ) - ]) - .unwrap(), + attesting_indices: VariableList::new(vec![0_u64; E::MaxValidatorsPerSlot::to_usize()]) + .unwrap(), data: AttestationData::default(), signature: AggregateSignature::empty(), }; @@ -626,12 +622,8 @@ impl> BeaconBlockElectra E::max_attester_slashings_electra() ] .into(); - // TODO(electra): check this let attestation = AttestationElectra { - aggregation_bits: BitList::with_capacity( - E::MaxValidatorsPerCommitteePerSlot::to_usize(), - ) - .unwrap(), + aggregation_bits: BitList::with_capacity(E::MaxValidatorsPerSlot::to_usize()).unwrap(), data: AttestationData::default(), signature: AggregateSignature::empty(), // TODO(electra): does this actually allocate the size correctly? diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 577f282a556..599c0bfc39c 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -159,6 +159,7 @@ pub enum Error { IndexNotSupported(usize), InvalidFlagIndex(usize), MerkleTreeError(merkle_proof::MerkleTreeError), + NoCommitteeFound, } /// Control whether an epoch-indexed field can be indexed at the next epoch or not. diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index e39b0dc9cf6..cec4db2da51 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -63,7 +63,7 @@ pub trait EthSpec: * Misc */ type MaxValidatorsPerCommittee: Unsigned + Clone + Sync + Send + Debug + PartialEq + Eq; - type MaxValidatorsPerCommitteePerSlot: Unsigned + Clone + Sync + Send + Debug + PartialEq + Eq; + type MaxValidatorsPerSlot: Unsigned + Clone + Sync + Send + Debug + PartialEq + Eq; type MaxCommitteesPerSlot: Unsigned + Clone + Sync + Send + Debug + PartialEq + Eq; /* * Time parameters @@ -352,7 +352,7 @@ impl EthSpec for MainnetEthSpec { type SubnetBitfieldLength = U64; type MaxValidatorsPerCommittee = U2048; type MaxCommitteesPerSlot = U64; - type MaxValidatorsPerCommitteePerSlot = U131072; + type MaxValidatorsPerSlot = U131072; type GenesisEpoch = U0; type SlotsPerEpoch = U32; type EpochsPerEth1VotingPeriod = U64; @@ -433,7 +433,7 @@ impl EthSpec for MinimalEthSpec { SyncCommitteeSubnetCount, MaxValidatorsPerCommittee, MaxCommitteesPerSlot, - MaxValidatorsPerCommitteePerSlot, + MaxValidatorsPerSlot, GenesisEpoch, HistoricalRootsLimit, ValidatorRegistryLimit, @@ -475,7 +475,7 @@ impl EthSpec for GnosisEthSpec { type SubnetBitfieldLength = U64; type MaxValidatorsPerCommittee = U2048; type MaxCommitteesPerSlot = U64; - type MaxValidatorsPerCommitteePerSlot = U131072; + type MaxValidatorsPerSlot = U131072; type GenesisEpoch = U0; type SlotsPerEpoch = U16; type EpochsPerEth1VotingPeriod = U64; diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index d17d00a3fa8..4e7e061b45f 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -59,7 +59,7 @@ pub struct IndexedAttestation { pub attesting_indices: VariableList, #[superstruct(only(Electra), partial_getter(rename = "attesting_indices_electra"))] #[serde(with = "quoted_variable_list_u64")] - pub attesting_indices: VariableList, + pub attesting_indices: VariableList, pub data: AttestationData, pub signature: AggregateSignature, } diff --git a/consensus/types/src/signed_aggregate_and_proof.rs b/consensus/types/src/signed_aggregate_and_proof.rs index 649b5fc6fac..57a2ce5babe 100644 --- a/consensus/types/src/signed_aggregate_and_proof.rs +++ b/consensus/types/src/signed_aggregate_and_proof.rs @@ -2,8 +2,8 @@ use super::{ AggregateAndProof, AggregateAndProofBase, AggregateAndProofElectra, AggregateAndProofRef, }; use super::{ - Attestation, ChainSpec, Domain, EthSpec, Fork, Hash256, SecretKey, SelectionProof, Signature, - SignedRoot, + AttestationRef, ChainSpec, Domain, EthSpec, Fork, Hash256, SecretKey, SelectionProof, + Signature, SignedRoot, }; use crate::test_utils::TestRandom; use serde::{Deserialize, Serialize}; @@ -58,7 +58,7 @@ impl SignedAggregateAndProof { /// If `selection_proof.is_none()` it will be computed locally. pub fn from_aggregate( aggregator_index: u64, - aggregate: Attestation, + aggregate: AttestationRef<'_, E>, selection_proof: Option, secret_key: &SecretKey, fork: &Fork, diff --git a/lcli/src/indexed_attestations.rs b/lcli/src/indexed_attestations.rs index 8d932ba7600..ccc14171128 100644 --- a/lcli/src/indexed_attestations.rs +++ b/lcli/src/indexed_attestations.rs @@ -1,6 +1,6 @@ use clap::ArgMatches; use clap_utils::parse_required; -use state_processing::common::get_indexed_attestation; +use state_processing::common::{attesting_indices_base, attesting_indices_electra}; use std::fs::File; use std::io::Read; use std::path::{Path, PathBuf}; @@ -33,9 +33,14 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { let indexed_attestations = attestations .into_iter() - .map(|att| { - let committee = state.get_beacon_committee(att.data().slot, att.data().index)?; - get_indexed_attestation(committee.committee, att.to_ref()) + .map(|att| match att { + Attestation::Base(att) => { + let committee = state.get_beacon_committee(att.data.slot, att.data.index)?; + attesting_indices_base::get_indexed_attestation(committee.committee, &att) + } + Attestation::Electra(att) => { + attesting_indices_electra::get_indexed_attestation_from_state(&state, &att) + } }) .collect::, _>>() .map_err(|e| format!("Error constructing indexed attestation: {:?}", e))?; diff --git a/slasher/src/attester_record.rs b/slasher/src/attester_record.rs index 8de25160725..1cd4ba7d4e0 100644 --- a/slasher/src/attester_record.rs +++ b/slasher/src/attester_record.rs @@ -80,7 +80,7 @@ impl IndexedAttesterRecord { #[derive(Debug, Clone, Encode, Decode, TreeHash)] struct IndexedAttestationHeader { - pub attesting_indices: VariableList, + pub attesting_indices: VariableList, pub data_root: Hash256, pub signature: AggregateSignature, } @@ -106,15 +106,15 @@ impl From> for AttesterRecord { #[cfg(test)] mod test { use super::*; - use crate::test_utils::indexed_att; + use crate::test_utils::indexed_att_electra; // Check correctness of fast hashing #[test] fn fast_hash() { let data = vec![ - indexed_att(vec![], 0, 0, 0), - indexed_att(vec![1, 2, 3], 12, 14, 1), - indexed_att(vec![4], 0, 5, u64::MAX), + indexed_att_electra(vec![], 0, 0, 0), + indexed_att_electra(vec![1, 2, 3], 12, 14, 1), + indexed_att_electra(vec![4], 0, 5, u64::MAX), ]; for att in data { assert_eq!( diff --git a/slasher/src/test_utils.rs b/slasher/src/test_utils.rs index 634b4d52113..8bbf042c2dd 100644 --- a/slasher/src/test_utils.rs +++ b/slasher/src/test_utils.rs @@ -1,12 +1,38 @@ use std::collections::HashSet; use types::{ - indexed_attestation::IndexedAttestationBase, AggregateSignature, AttestationData, - AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BeaconBlockHeader, Checkpoint, - Epoch, Hash256, IndexedAttestation, MainnetEthSpec, Signature, SignedBeaconBlockHeader, Slot, + indexed_attestation::{IndexedAttestationBase, IndexedAttestationElectra}, + AggregateSignature, AttestationData, AttesterSlashing, AttesterSlashingBase, + AttesterSlashingElectra, BeaconBlockHeader, Checkpoint, Epoch, Hash256, IndexedAttestation, + MainnetEthSpec, Signature, SignedBeaconBlockHeader, Slot, }; pub type E = MainnetEthSpec; +pub fn indexed_att_electra( + attesting_indices: impl AsRef<[u64]>, + source_epoch: u64, + target_epoch: u64, + target_root: u64, +) -> IndexedAttestation { + IndexedAttestation::Electra(IndexedAttestationElectra { + attesting_indices: attesting_indices.as_ref().to_vec().into(), + data: AttestationData { + slot: Slot::new(0), + index: 0, + beacon_block_root: Hash256::zero(), + source: Checkpoint { + epoch: Epoch::new(source_epoch), + root: Hash256::from_low_u64_be(0), + }, + target: Checkpoint { + epoch: Epoch::new(target_epoch), + root: Hash256::from_low_u64_be(target_root), + }, + }, + signature: AggregateSignature::empty(), + }) +} + pub fn indexed_att( attesting_indices: impl AsRef<[u64]>, source_epoch: u64, diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index 2d5ea4149ef..c1e7386bc45 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -229,6 +229,10 @@ impl SszStaticHandler { Self::for_forks(vec![ForkName::Deneb]) } + pub fn electra_only() -> Self { + Self::for_forks(vec![ForkName::Electra]) + } + pub fn altair_and_later() -> Self { Self::for_forks(ForkName::list_all()[1..].to_vec()) } @@ -240,6 +244,10 @@ impl SszStaticHandler { pub fn capella_and_later() -> Self { Self::for_forks(ForkName::list_all()[3..].to_vec()) } + + pub fn pre_electra() -> Self { + Self::for_forks(ForkName::list_all()[0..5].to_vec()) + } } /// Handler for SSZ types that implement `CachedTreeHash`. diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 49ebbe81909..96eaffc75b5 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -37,11 +37,14 @@ macro_rules! type_name_generic { type_name!(MinimalEthSpec, "minimal"); type_name!(MainnetEthSpec, "mainnet"); - type_name_generic!(AggregateAndProof); +type_name_generic!(AggregateAndProofBase, "AggregateAndProof"); +type_name_generic!(AggregateAndProofElectra, "AggregateAndProof"); type_name_generic!(Attestation); type_name!(AttestationData); type_name_generic!(AttesterSlashing); +type_name_generic!(AttesterSlashingBase, "AttesterSlashing"); +type_name_generic!(AttesterSlashingElectra, "AttesterSlashing"); type_name_generic!(BeaconBlock); type_name_generic!(BeaconBlockBody); type_name_generic!(BeaconBlockBodyBase, "BeaconBlockBody"); @@ -108,6 +111,8 @@ type_name_generic!(LightClientUpdateDeneb, "LightClientUpdate"); type_name_generic!(PendingAttestation); type_name!(ProposerSlashing); type_name_generic!(SignedAggregateAndProof); +type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProofBase"); +type_name_generic!(SignedAggregateAndProofElectra, "SignedAggregateAndProofElectra"); type_name_generic!(SignedBeaconBlock); type_name!(SignedBeaconBlockHeader); type_name_generic!(SignedContributionAndProof); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 5226c7ac2b0..669c7721748 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -217,12 +217,11 @@ mod ssz_static { use ef_tests::{Handler, SszStaticHandler, SszStaticTHCHandler, SszStaticWithSpecHandler}; use types::blob_sidecar::BlobIdentifier; use types::historical_summary::HistoricalSummary; - use types::{LightClientBootstrapAltair, *}; + use types::{AttesterSlashingBase, AttesterSlashingElectra, LightClientBootstrapAltair, *}; ssz_static_test!(aggregate_and_proof, AggregateAndProof<_>); ssz_static_test!(attestation, Attestation<_>); ssz_static_test!(attestation_data, AttestationData); - ssz_static_test!(attester_slashing, AttesterSlashing<_>); ssz_static_test!(beacon_block, SszStaticWithSpecHandler, BeaconBlock<_>); ssz_static_test!(beacon_block_header, BeaconBlockHeader); ssz_static_test!(beacon_state, SszStaticTHCHandler, BeaconState<_>); @@ -238,7 +237,6 @@ mod ssz_static { ssz_static_test!(indexed_attestation, IndexedAttestation<_>); ssz_static_test!(pending_attestation, PendingAttestation<_>); ssz_static_test!(proposer_slashing, ProposerSlashing); - ssz_static_test!(signed_aggregate_and_proof, SignedAggregateAndProof<_>); ssz_static_test!( signed_beacon_block, SszStaticWithSpecHandler, @@ -249,6 +247,24 @@ mod ssz_static { ssz_static_test!(signing_data, SigningData); ssz_static_test!(validator, Validator); ssz_static_test!(voluntary_exit, VoluntaryExit); + + + #[test] + fn signed_aggregate_and_proof() { + SszStaticHandler::, MinimalEthSpec>::pre_electra( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::pre_electra( + ) + .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); + } + // BeaconBlockBody has no internal indicator of which fork it is for, so we test it separately. #[test] fn beacon_block_body() { @@ -272,6 +288,22 @@ mod ssz_static { .run(); } + #[test] + fn signed_aggregate_and_proof() { + SszStaticHandler::, MinimalEthSpec>::pre_electra( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::pre_electra( + ) + .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); + } + // Altair and later #[test] fn contribution_and_proof() { From f30246b9d46d4c40ff1f5a1ed276f490752b2b14 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Wed, 8 May 2024 11:40:08 -0500 Subject: [PATCH 023/227] Some small changes (#5739) --- beacon_node/beacon_chain/src/attestation_verification.rs | 7 ++++--- beacon_node/operation_pool/src/attestation_storage.rs | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 28091847683..62e65d5f87a 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -560,6 +560,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { })?; if let Some(committee) = committee { + // TODO(electra): // Note: this clones the signature which is known to be a relatively slow operation. // // Future optimizations should remove this clone. @@ -599,7 +600,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { let indexed_attestation = match map_attestation_committees( chain, - &attestation, + attestation, get_indexed_attestation_with_committee, ) { Ok(indexed_attestation) => indexed_attestation, @@ -1281,7 +1282,7 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( chain: &BeaconChain, attestation: AttestationRef, ) -> Result<(IndexedAttestation, CommitteesPerSlot), Error> { - map_attestation_committees(chain, &attestation, |(committees, committees_per_slot)| { + map_attestation_committees(chain, attestation, |(committees, committees_per_slot)| { match attestation { AttestationRef::Base(att) => { let committee = committees @@ -1335,7 +1336,7 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( /// from disk and then update the `shuffling_cache`. fn map_attestation_committees( chain: &BeaconChain, - attestation: &AttestationRef, + attestation: AttestationRef, map_fn: F, ) -> Result where diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index 7d7a72704d9..00fbcbe4b01 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -239,6 +239,7 @@ impl AttestationMap { let attestation_map = self.checkpoint_map.entry(checkpoint).or_default(); let attestations = attestation_map.attestations.entry(data).or_default(); + // TODO(electra): // Greedily aggregate the attestation with all existing attestations. // NOTE: this is sub-optimal and in future we will remove this in favour of max-clique // aggregation. From 43c3f63e3000e2a5d8b486180380d884b5980edc Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Wed, 8 May 2024 11:53:08 -0500 Subject: [PATCH 024/227] cargo fmt (#5740) --- consensus/types/src/attestation.rs | 2 +- testing/ef_tests/src/type_name.rs | 5 ++++- testing/ef_tests/tests/tests.rs | 21 ++++++++------------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 61e63e11e22..bcecfde10e4 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -437,7 +437,7 @@ mod tests { assert_eq!(signature, 288 + 16); let attestation_expected = aggregation_bits + attestation_data + signature; - // TODO(electra) since we've removed attestation aggregation for electra variant + // TODO(electra) since we've removed attestation aggregation for electra variant // i've updated the attestation value expected from 488 544 // assert_eq!(attestation_expected, 488); assert_eq!(attestation_expected, 488); diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 96eaffc75b5..30db5c0e4a9 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -112,7 +112,10 @@ type_name_generic!(PendingAttestation); type_name!(ProposerSlashing); type_name_generic!(SignedAggregateAndProof); type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProofBase"); -type_name_generic!(SignedAggregateAndProofElectra, "SignedAggregateAndProofElectra"); +type_name_generic!( + SignedAggregateAndProofElectra, + "SignedAggregateAndProofElectra" +); type_name_generic!(SignedBeaconBlock); type_name!(SignedBeaconBlockHeader); type_name_generic!(SignedContributionAndProof); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 669c7721748..85d9362aaeb 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -248,21 +248,16 @@ mod ssz_static { ssz_static_test!(validator, Validator); ssz_static_test!(voluntary_exit, VoluntaryExit); - #[test] fn signed_aggregate_and_proof() { - SszStaticHandler::, MinimalEthSpec>::pre_electra( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::pre_electra( - ) - .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( - ) - .run(); + SszStaticHandler::, MinimalEthSpec>::pre_electra() + .run(); + SszStaticHandler::, MainnetEthSpec>::pre_electra() + .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only() + .run(); } // BeaconBlockBody has no internal indicator of which fork it is for, so we test it separately. From 7abb7621d5798ba4b5c6616d0cfdcc0af6e851af Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 8 May 2024 14:11:22 -0400 Subject: [PATCH 025/227] fix attestation verification --- .../per_block_processing/verify_attestation.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/verify_attestation.rs b/consensus/state_processing/src/per_block_processing/verify_attestation.rs index 6bfb5d7cfe7..6bfb51d475b 100644 --- a/consensus/state_processing/src/per_block_processing/verify_attestation.rs +++ b/consensus/state_processing/src/per_block_processing/verify_attestation.rs @@ -68,10 +68,20 @@ pub fn verify_attestation_for_state<'ctxt, E: EthSpec>( ) -> Result> { let data = attestation.data(); - verify!( - data.index < state.get_committee_count_at_slot(data.slot)?, - Invalid::BadCommitteeIndex - ); + // TODO(electra) choosing a validation based on the attestation's fork + // rather than the state's fork makes this simple, but technically the spec + // defines this verification based on the state's fork. + match attestation { + AttestationRef::Base(_) => { + verify!( + data.index < state.get_committee_count_at_slot(data.slot)?, + Invalid::BadCommitteeIndex + ); + } + AttestationRef::Electra(_) => { + verify!(data.index == 0, Invalid::BadCommitteeIndex); + } + } // Verify the Casper FFG vote. verify_casper_ffg_vote(attestation, state)?; From 42a499373fe1bc72a2c5885652827c6c00f7a135 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Mon, 6 May 2024 15:48:46 -0700 Subject: [PATCH 026/227] Add new engine api methods --- beacon_node/execution_layer/src/engine_api.rs | 11 +- .../execution_layer/src/engine_api/http.rs | 52 +++++++-- .../src/engine_api/json_structures.rs | 102 +++++++++++++++++- .../execution_layer/src/test_utils/mod.rs | 2 + 4 files changed, 155 insertions(+), 12 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index ce1e0fec5dd..6f6ec8b8947 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -3,7 +3,8 @@ use crate::http::{ ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_FORKCHOICE_UPDATED_V3, ENGINE_GET_CLIENT_VERSION_V1, ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, - ENGINE_GET_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, + ENGINE_GET_PAYLOAD_V3, ENGINE_GET_PAYLOAD_V4, ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, + ENGINE_NEW_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V4, }; use eth2::types::{ BlobsBundle, SsePayloadAttributes, SsePayloadAttributesV1, SsePayloadAttributesV2, @@ -652,6 +653,7 @@ pub struct EngineCapabilities { pub new_payload_v1: bool, pub new_payload_v2: bool, pub new_payload_v3: bool, + pub new_payload_v4: bool, pub forkchoice_updated_v1: bool, pub forkchoice_updated_v2: bool, pub forkchoice_updated_v3: bool, @@ -660,6 +662,7 @@ pub struct EngineCapabilities { pub get_payload_v1: bool, pub get_payload_v2: bool, pub get_payload_v3: bool, + pub get_payload_v4: bool, pub get_client_version_v1: bool, } @@ -675,6 +678,9 @@ impl EngineCapabilities { if self.new_payload_v3 { response.push(ENGINE_NEW_PAYLOAD_V3); } + if self.new_payload_v4 { + response.push(ENGINE_NEW_PAYLOAD_V4); + } if self.forkchoice_updated_v1 { response.push(ENGINE_FORKCHOICE_UPDATED_V1); } @@ -699,6 +705,9 @@ impl EngineCapabilities { if self.get_payload_v3 { response.push(ENGINE_GET_PAYLOAD_V3); } + if self.get_payload_v4 { + response.push(ENGINE_GET_PAYLOAD_V4); + } if self.get_client_version_v1 { response.push(ENGINE_GET_CLIENT_VERSION_V1); } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 93705a16925..e81027340f6 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -34,11 +34,13 @@ pub const ETH_SYNCING_TIMEOUT: Duration = Duration::from_secs(1); pub const ENGINE_NEW_PAYLOAD_V1: &str = "engine_newPayloadV1"; pub const ENGINE_NEW_PAYLOAD_V2: &str = "engine_newPayloadV2"; pub const ENGINE_NEW_PAYLOAD_V3: &str = "engine_newPayloadV3"; +pub const ENGINE_NEW_PAYLOAD_V4: &str = "engine_newPayloadV4"; pub const ENGINE_NEW_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(8); pub const ENGINE_GET_PAYLOAD_V1: &str = "engine_getPayloadV1"; pub const ENGINE_GET_PAYLOAD_V2: &str = "engine_getPayloadV2"; pub const ENGINE_GET_PAYLOAD_V3: &str = "engine_getPayloadV3"; +pub const ENGINE_GET_PAYLOAD_V4: &str = "engine_getPayloadV4"; pub const ENGINE_GET_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(2); pub const ENGINE_FORKCHOICE_UPDATED_V1: &str = "engine_forkchoiceUpdatedV1"; @@ -66,9 +68,11 @@ pub static LIGHTHOUSE_CAPABILITIES: &[&str] = &[ ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, + ENGINE_NEW_PAYLOAD_V4, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, ENGINE_GET_PAYLOAD_V3, + ENGINE_GET_PAYLOAD_V4, ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_FORKCHOICE_UPDATED_V3, @@ -833,7 +837,7 @@ impl HttpJsonRpc { Ok(response.into()) } - pub async fn new_payload_v3_electra( + pub async fn new_payload_v4_electra( &self, new_payload_request_electra: NewPayloadRequestElectra<'_, E>, ) -> Result { @@ -845,7 +849,7 @@ impl HttpJsonRpc { let response: JsonPayloadStatusV1 = self .rpc_request( - ENGINE_NEW_PAYLOAD_V3, + ENGINE_NEW_PAYLOAD_V4, params, ENGINE_NEW_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, ) @@ -929,10 +933,40 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V3(response).into()) } + ForkName::Base + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + // TODO(pawan): make sure electra should not support getpayloadv3 + | ForkName::Electra => Err(Error::UnsupportedForkVariant(format!( + "called get_payload_v3 with {}", + fork_name + ))), + } + } + + pub async fn get_payload_v4( + &self, + fork_name: ForkName, + payload_id: PayloadId, + ) -> Result, Error> { + let params = json!([JsonPayloadIdRequest::from(payload_id)]); + + match fork_name { + ForkName::Deneb => { + let response: JsonGetPayloadResponseV3 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V4, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + Ok(JsonGetPayloadResponse::V3(response).into()) + } ForkName::Electra => { let response: JsonGetPayloadResponseV4 = self .rpc_request( - ENGINE_GET_PAYLOAD_V3, + ENGINE_GET_PAYLOAD_V4, params, ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, ) @@ -940,7 +974,7 @@ impl HttpJsonRpc { Ok(JsonGetPayloadResponse::V4(response).into()) } ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => Err( - Error::UnsupportedForkVariant(format!("called get_payload_v3 with {}", fork_name)), + Error::UnsupportedForkVariant(format!("called get_payload_v4 with {}", fork_name)), ), } } @@ -1067,6 +1101,7 @@ impl HttpJsonRpc { new_payload_v1: capabilities.contains(ENGINE_NEW_PAYLOAD_V1), new_payload_v2: capabilities.contains(ENGINE_NEW_PAYLOAD_V2), new_payload_v3: capabilities.contains(ENGINE_NEW_PAYLOAD_V3), + new_payload_v4: capabilities.contains(ENGINE_NEW_PAYLOAD_V4), forkchoice_updated_v1: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V1), forkchoice_updated_v2: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V2), forkchoice_updated_v3: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V3), @@ -1077,6 +1112,7 @@ impl HttpJsonRpc { get_payload_v1: capabilities.contains(ENGINE_GET_PAYLOAD_V1), get_payload_v2: capabilities.contains(ENGINE_GET_PAYLOAD_V2), get_payload_v3: capabilities.contains(ENGINE_GET_PAYLOAD_V3), + get_payload_v4: capabilities.contains(ENGINE_GET_PAYLOAD_V4), get_client_version_v1: capabilities.contains(ENGINE_GET_CLIENT_VERSION_V1), }) } @@ -1200,7 +1236,7 @@ impl HttpJsonRpc { } NewPayloadRequest::Electra(new_payload_request_electra) => { if engine_capabilities.new_payload_v3 { - self.new_payload_v3_electra(new_payload_request_electra) + self.new_payload_v4_electra(new_payload_request_electra) .await } else { Err(Error::RequiredMethodUnsupported("engine_newPayloadV3")) @@ -1221,14 +1257,16 @@ impl HttpJsonRpc { ForkName::Bellatrix | ForkName::Capella => { if engine_capabilities.get_payload_v2 { self.get_payload_v2(fork_name, payload_id).await - } else if engine_capabilities.new_payload_v1 { + } else if engine_capabilities.get_payload_v1 { self.get_payload_v1(payload_id).await } else { Err(Error::RequiredMethodUnsupported("engine_getPayload")) } } ForkName::Deneb | ForkName::Electra => { - if engine_capabilities.get_payload_v3 { + if engine_capabilities.get_payload_v4 { + self.get_payload_v4(fork_name, payload_id).await + } else if engine_capabilities.get_payload_v3 { self.get_payload_v3(fork_name, payload_id).await } else { Err(Error::RequiredMethodUnsupported("engine_getPayloadV3")) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 50d3519e129..822a3802a85 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -4,7 +4,10 @@ use strum::EnumString; use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::BlobsList; -use types::{FixedVector, Unsigned}; +use types::{ + DepositReceipt, ExecutionLayerWithdrawalRequest, FixedVector, PublicKeyBytes, Signature, + Unsigned, +}; #[derive(Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -101,6 +104,11 @@ pub struct JsonExecutionPayload { #[superstruct(only(V3, V4))] #[serde(with = "serde_utils::u64_hex_be")] pub excess_blob_gas: u64, + #[superstruct(only(V4))] + pub deposit_requests: VariableList, + #[superstruct(only(V4))] + pub withdrawal_requests: + VariableList, } impl From> for JsonExecutionPayloadV1 { @@ -203,6 +211,18 @@ impl From> for JsonExecutionPayloadV4 .into(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, + deposit_requests: payload + .deposit_receipts + .into_iter() + .map(Into::into) + .collect::>() + .into(), + withdrawal_requests: payload + .withdrawal_requests + .into_iter() + .map(Into::into) + .collect::>() + .into(), } } } @@ -319,9 +339,18 @@ impl From> for ExecutionPayloadElectra .into(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, - // TODO(electra) - deposit_receipts: Default::default(), - withdrawal_requests: Default::default(), + deposit_receipts: payload + .deposit_requests + .into_iter() + .map(Into::into) + .collect::>() + .into(), + withdrawal_requests: payload + .withdrawal_requests + .into_iter() + .map(Into::into) + .collect::>() + .into(), } } } @@ -783,3 +812,68 @@ impl TryFrom for ClientVersionV1 { }) } } + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct JsonDepositRequest { + pub pubkey: PublicKeyBytes, + pub withdrawal_credentials: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub amount: u64, + pub signature: Signature, + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, +} + +impl From for JsonDepositRequest { + fn from(deposit: DepositReceipt) -> Self { + Self { + pubkey: deposit.pubkey, + withdrawal_credentials: deposit.withdrawal_credentials, + amount: deposit.amount, + signature: deposit.signature, + index: deposit.index, + } + } +} + +impl From for DepositReceipt { + fn from(json_deposit: JsonDepositRequest) -> Self { + Self { + pubkey: json_deposit.pubkey, + withdrawal_credentials: json_deposit.withdrawal_credentials, + amount: json_deposit.amount, + signature: json_deposit.signature, + index: json_deposit.index, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct JsonWithdrawalRequest { + pub source_address: Address, + pub validator_public_key: PublicKeyBytes, + #[serde(with = "serde_utils::quoted_u64")] + pub amount: u64, +} + +impl From for JsonWithdrawalRequest { + fn from(withdrawal_request: ExecutionLayerWithdrawalRequest) -> Self { + Self { + source_address: withdrawal_request.source_address, + validator_public_key: withdrawal_request.validator_pubkey, + amount: withdrawal_request.amount, + } + } +} + +impl From for ExecutionLayerWithdrawalRequest { + fn from(json_withdrawal_request: JsonWithdrawalRequest) -> Self { + Self { + source_address: json_withdrawal_request.source_address, + validator_pubkey: json_withdrawal_request.validator_public_key, + amount: json_withdrawal_request.amount, + } + } +} diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index a6d47995af8..7b00ca9fbc3 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -43,6 +43,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { new_payload_v1: true, new_payload_v2: true, new_payload_v3: true, + new_payload_v4: true, forkchoice_updated_v1: true, forkchoice_updated_v2: true, forkchoice_updated_v3: true, @@ -51,6 +52,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { get_payload_v1: true, get_payload_v2: true, get_payload_v3: true, + get_payload_v4: true, get_client_version_v1: true, }; From ca2a9461759e913dbd1902f55902820cf7415972 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 7 May 2024 14:42:51 -0700 Subject: [PATCH 027/227] Fix the versioning of v4 requests --- .../execution_layer/src/engine_api/http.rs | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index e81027340f6..aea87dd7d41 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -937,7 +937,6 @@ impl HttpJsonRpc { | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella - // TODO(pawan): make sure electra should not support getpayloadv3 | ForkName::Electra => Err(Error::UnsupportedForkVariant(format!( "called get_payload_v3 with {}", fork_name @@ -953,16 +952,6 @@ impl HttpJsonRpc { let params = json!([JsonPayloadIdRequest::from(payload_id)]); match fork_name { - ForkName::Deneb => { - let response: JsonGetPayloadResponseV3 = self - .rpc_request( - ENGINE_GET_PAYLOAD_V4, - params, - ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, - ) - .await?; - Ok(JsonGetPayloadResponse::V3(response).into()) - } ForkName::Electra => { let response: JsonGetPayloadResponseV4 = self .rpc_request( @@ -973,9 +962,14 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V4(response).into()) } - ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => Err( - Error::UnsupportedForkVariant(format!("called get_payload_v4 with {}", fork_name)), - ), + ForkName::Base + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => Err(Error::UnsupportedForkVariant(format!( + "called get_payload_v4 with {}", + fork_name + ))), } } @@ -1235,11 +1229,11 @@ impl HttpJsonRpc { } } NewPayloadRequest::Electra(new_payload_request_electra) => { - if engine_capabilities.new_payload_v3 { + if engine_capabilities.new_payload_v4 { self.new_payload_v4_electra(new_payload_request_electra) .await } else { - Err(Error::RequiredMethodUnsupported("engine_newPayloadV3")) + Err(Error::RequiredMethodUnsupported("engine_newPayloadV4")) } } } @@ -1263,13 +1257,18 @@ impl HttpJsonRpc { Err(Error::RequiredMethodUnsupported("engine_getPayload")) } } - ForkName::Deneb | ForkName::Electra => { + ForkName::Deneb => { + if engine_capabilities.get_payload_v3 { + self.get_payload_v3(fork_name, payload_id).await + } else { + Err(Error::RequiredMethodUnsupported("engine_getPayloadv3")) + } + } + ForkName::Electra => { if engine_capabilities.get_payload_v4 { self.get_payload_v4(fork_name, payload_id).await - } else if engine_capabilities.get_payload_v3 { - self.get_payload_v3(fork_name, payload_id).await } else { - Err(Error::RequiredMethodUnsupported("engine_getPayloadV3")) + Err(Error::RequiredMethodUnsupported("engine_getPayloadv4")) } } ForkName::Base | ForkName::Altair => Err(Error::UnsupportedForkVariant(format!( From 1ddd078d324fde76ca178a97321e22e569b5ff4f Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 7 May 2024 16:13:36 -0700 Subject: [PATCH 028/227] Handle new engine api methods in mock EL --- .../src/test_utils/handle_rpc.rs | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 1dc8f0ab83e..5e31646d31a 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -111,20 +111,11 @@ pub async fn handle_rpc( .map(|jep| JsonExecutionPayload::V1(jep)) }) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, - ENGINE_NEW_PAYLOAD_V3 => get_param::>(params, 0) + ENGINE_NEW_PAYLOAD_V3 => get_param::>(params, 0) + .map(|jep| JsonExecutionPayload::V3(jep)) + .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, + ENGINE_NEW_PAYLOAD_V4 => get_param::>(params, 0) .map(|jep| JsonExecutionPayload::V4(jep)) - .or_else(|_| { - get_param::>(params, 0) - .map(|jep| JsonExecutionPayload::V3(jep)) - .or_else(|_| { - get_param::>(params, 0) - .map(|jep| JsonExecutionPayload::V2(jep)) - .or_else(|_| { - get_param::>(params, 0) - .map(|jep| JsonExecutionPayload::V1(jep)) - }) - }) - }) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, _ => unreachable!(), }; @@ -190,7 +181,10 @@ pub async fn handle_rpc( } } ForkName::Electra => { - if method == ENGINE_NEW_PAYLOAD_V1 || method == ENGINE_NEW_PAYLOAD_V2 { + if method == ENGINE_NEW_PAYLOAD_V1 + || method == ENGINE_NEW_PAYLOAD_V2 + || method == ENGINE_NEW_PAYLOAD_V3 + { return Err(( format!("{} called after Electra fork!", method), GENERIC_ERROR_CODE, @@ -259,7 +253,10 @@ pub async fn handle_rpc( Ok(serde_json::to_value(JsonPayloadStatusV1::from(response)).unwrap()) } - ENGINE_GET_PAYLOAD_V1 | ENGINE_GET_PAYLOAD_V2 | ENGINE_GET_PAYLOAD_V3 => { + ENGINE_GET_PAYLOAD_V1 + | ENGINE_GET_PAYLOAD_V2 + | ENGINE_GET_PAYLOAD_V3 + | ENGINE_GET_PAYLOAD_V4 => { let request: JsonPayloadIdRequest = get_param(params, 0).map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?; let id = request.into(); @@ -309,7 +306,9 @@ pub async fn handle_rpc( .read() .get_fork_at_timestamp(response.timestamp()) == ForkName::Electra - && method == ENGINE_GET_PAYLOAD_V1 + && (method == ENGINE_GET_PAYLOAD_V1 + || method == ENGINE_GET_PAYLOAD_V2 + || method == ENGINE_GET_PAYLOAD_V3) { return Err(( format!("{} called after Electra fork!", method), @@ -353,6 +352,9 @@ pub async fn handle_rpc( }) .unwrap() } + _ => unreachable!(), + }), + ENGINE_GET_PAYLOAD_V4 => Ok(match JsonExecutionPayload::from(response) { JsonExecutionPayload::V4(execution_payload) => { serde_json::to_value(JsonGetPayloadResponseV4 { execution_payload, From 3ef7c9078e4a06b097963fdad51087ea7f869bc6 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 7 May 2024 16:21:21 -0700 Subject: [PATCH 029/227] Note todo --- beacon_node/execution_layer/src/engine_api/json_structures.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 822a3802a85..c9a7c072f8c 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -105,6 +105,7 @@ pub struct JsonExecutionPayload { #[serde(with = "serde_utils::u64_hex_be")] pub excess_blob_gas: u64, #[superstruct(only(V4))] + // TODO(electra): Field name should be changed post devnet-0. See https://github.com/ethereum/execution-apis/pull/544 pub deposit_requests: VariableList, #[superstruct(only(V4))] pub withdrawal_requests: From 683de56f6e9185a2d091aff92db9dae2d66d117e Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Wed, 8 May 2024 13:42:29 -0700 Subject: [PATCH 030/227] Fix todos --- beacon_node/execution_layer/src/engine_api.rs | 18 +++++++++++++++ beacon_node/execution_layer/src/lib.rs | 23 +++++++++++++++---- .../types/src/execution_payload_header.rs | 1 - 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 6f6ec8b8947..df6b30baa2c 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -41,6 +41,8 @@ pub use new_payload_request::{ NewPayloadRequestDeneb, NewPayloadRequestElectra, }; +use self::json_structures::{JsonDepositRequest, JsonWithdrawalRequest}; + pub const LATEST_TAG: &str = "latest"; pub type PayloadId = [u8; 8]; @@ -64,6 +66,8 @@ pub enum Error { TransitionConfigurationMismatch, SszError(ssz_types::Error), DeserializeWithdrawals(ssz_types::Error), + DeserializeDepositReceipts(ssz_types::Error), + DeserializeWithdrawalRequests(ssz_types::Error), BuilderApi(builder_client::Error), IncorrectStateVariant, RequiredMethodUnsupported(&'static str), @@ -198,6 +202,10 @@ pub struct ExecutionBlockWithTransactions { #[superstruct(only(Deneb, Electra))] #[serde(with = "serde_utils::u64_hex_be")] pub excess_blob_gas: u64, + #[superstruct(only(Electra))] + pub deposit_receipts: Vec, + #[superstruct(only(Electra))] + pub withdrawal_requests: Vec, } impl TryFrom> for ExecutionBlockWithTransactions { @@ -305,6 +313,16 @@ impl TryFrom> for ExecutionBlockWithTransactions .collect(), blob_gas_used: block.blob_gas_used, excess_blob_gas: block.excess_blob_gas, + deposit_receipts: block + .deposit_receipts + .into_iter() + .map(|deposit| deposit.into()) + .collect(), + withdrawal_requests: block + .withdrawal_requests + .into_iter() + .map(|withdrawal| withdrawal.into()) + .collect(), }) } }; diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 11329259e90..15a1c194629 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1985,6 +1985,22 @@ impl ExecutionLayer { .collect(), ) .map_err(ApiError::DeserializeWithdrawals)?; + let deposit_receipts = VariableList::new( + electra_block + .deposit_receipts + .into_iter() + .map(Into::into) + .collect(), + ) + .map_err(ApiError::DeserializeDepositReceipts)?; + let withdrawal_requests = VariableList::new( + electra_block + .withdrawal_requests + .into_iter() + .map(Into::into) + .collect(), + ) + .map_err(ApiError::DeserializeWithdrawalRequests)?; ExecutionPayload::Electra(ExecutionPayloadElectra { parent_hash: electra_block.parent_hash, fee_recipient: electra_block.fee_recipient, @@ -2003,11 +2019,8 @@ impl ExecutionLayer { withdrawals, blob_gas_used: electra_block.blob_gas_used, excess_blob_gas: electra_block.excess_blob_gas, - // TODO(electra) - // deposit_receipts: electra_block.deposit_receipts, - // withdrawal_requests: electra_block.withdrawal_requests, - deposit_receipts: <_>::default(), - withdrawal_requests: <_>::default(), + deposit_receipts, + withdrawal_requests, }) } }; diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 2d7bc950714..a3df69652c2 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -117,7 +117,6 @@ impl ExecutionPayloadHeader { #[allow(clippy::arithmetic_side_effects)] pub fn ssz_max_var_len_for_fork(fork_name: ForkName) -> usize { // Matching here in case variable fields are added in future forks. - // TODO(electra): review electra changes match fork_name { ForkName::Base | ForkName::Altair From dd5c9a8c8160e5cf744687b321293312469dedba Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Wed, 8 May 2024 16:22:01 -0700 Subject: [PATCH 031/227] Add support for electra fields in getPayloadBodies --- beacon_node/execution_layer/src/engine_api.rs | 60 ++++++++++--------- .../src/engine_api/json_structures.rs | 19 ++++++ .../src/test_utils/handle_rpc.rs | 8 +++ consensus/types/src/execution_payload.rs | 4 ++ 4 files changed, 64 insertions(+), 27 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index df6b30baa2c..09e36be41f4 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -20,6 +20,7 @@ use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use strum::IntoStaticStr; use superstruct::superstruct; +use types::execution_payload::{DepositReceipts, WithdrawalRequests}; pub use types::{ Address, BeaconBlockRef, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, @@ -545,6 +546,8 @@ impl GetPayloadResponse { pub struct ExecutionPayloadBodyV1 { pub transactions: Transactions, pub withdrawals: Option>, + pub deposit_receipts: Option>, + pub withdrawal_requests: Option>, } impl ExecutionPayloadBodyV1 { @@ -632,35 +635,38 @@ impl ExecutionPayloadBodyV1 { } } ExecutionPayloadHeader::Electra(header) => { - if let Some(withdrawals) = self.withdrawals { - Ok(ExecutionPayload::Electra(ExecutionPayloadElectra { - parent_hash: header.parent_hash, - fee_recipient: header.fee_recipient, - state_root: header.state_root, - receipts_root: header.receipts_root, - logs_bloom: header.logs_bloom, - prev_randao: header.prev_randao, - block_number: header.block_number, - gas_limit: header.gas_limit, - gas_used: header.gas_used, - timestamp: header.timestamp, - extra_data: header.extra_data, - base_fee_per_gas: header.base_fee_per_gas, - block_hash: header.block_hash, - transactions: self.transactions, - withdrawals, - blob_gas_used: header.blob_gas_used, - excess_blob_gas: header.excess_blob_gas, - // TODO(electra) - deposit_receipts: <_>::default(), - withdrawal_requests: <_>::default(), - })) - } else { - Err(format!( - "block {} is post-capella but payload body doesn't have withdrawals", + let (Some(withdrawals), Some(deposit_receipts), Some(withdrawal_requests)) = ( + self.withdrawals, + self.deposit_receipts, + self.withdrawal_requests, + ) else { + return Err(format!( + "block {} is post-electra but payload body doesn't have withdrawals/deposit_receipts/withdrawal_requests \ + Check that ELs are returning receipts and withdrawal_requests in getPayloadBody requests", header.block_hash )) - } + }; + Ok(ExecutionPayload::Electra(ExecutionPayloadElectra { + parent_hash: header.parent_hash, + fee_recipient: header.fee_recipient, + state_root: header.state_root, + receipts_root: header.receipts_root, + logs_bloom: header.logs_bloom, + prev_randao: header.prev_randao, + block_number: header.block_number, + gas_limit: header.gas_limit, + gas_used: header.gas_used, + timestamp: header.timestamp, + extra_data: header.extra_data, + base_fee_per_gas: header.base_fee_per_gas, + block_hash: header.block_hash, + transactions: self.transactions, + withdrawals, + blob_gas_used: header.blob_gas_used, + excess_blob_gas: header.excess_blob_gas, + deposit_receipts, + withdrawal_requests, + })) } } } diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index c9a7c072f8c..28eb90fd1fa 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -724,6 +724,9 @@ pub struct JsonExecutionPayloadBodyV1 { #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, pub withdrawals: Option>, + pub deposit_receipts: Option>, + pub withdrawal_requests: + Option>, } impl From> for ExecutionPayloadBodyV1 { @@ -738,6 +741,22 @@ impl From> for ExecutionPayloadBodyV1< .collect::>(), ) }), + deposit_receipts: value.deposit_receipts.map(|json_receipts| { + DepositReceipts::::from( + json_receipts + .into_iter() + .map(Into::into) + .collect::>(), + ) + }), + withdrawal_requests: value.withdrawal_requests.map(|json_withdrawal_requests| { + WithdrawalRequests::::from( + json_withdrawal_requests + .into_iter() + .map(Into::into) + .collect::>(), + ) + }), } } } diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 5e31646d31a..d2c16da7998 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -580,6 +580,14 @@ pub async fn handle_rpc( .withdrawals() .ok() .map(|withdrawals| VariableList::from(withdrawals.clone())), + deposit_receipts: block.deposit_receipts().ok().map( + |deposit_receipts| VariableList::from(deposit_receipts.clone()), + ), + withdrawal_requests: block.withdrawal_requests().ok().map( + |withdrawal_requests| { + VariableList::from(withdrawal_requests.clone()) + }, + ), })); } None => response.push(None), diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 0946b9ecffa..cd6667b3751 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -13,6 +13,10 @@ pub type Transactions = VariableList< >; pub type Withdrawals = VariableList::MaxWithdrawalsPerPayload>; +pub type DepositReceipts = + VariableList::MaxDepositReceiptsPerPayload>; +pub type WithdrawalRequests = + VariableList::MaxWithdrawalRequestsPerPayload>; #[superstruct( variants(Bellatrix, Capella, Deneb, Electra), From 5e1d5ff641b8eed551dc82e57a46a9db26461b0c Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Wed, 8 May 2024 16:46:23 -0700 Subject: [PATCH 032/227] Add comments for potential versioning confusion --- beacon_node/execution_layer/src/test_utils/handle_rpc.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index d2c16da7998..b9d9d76186a 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -111,6 +111,9 @@ pub async fn handle_rpc( .map(|jep| JsonExecutionPayload::V1(jep)) }) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, + // From v3 onwards, we use the newPayload version only for the corresponding + // ExecutionPayload version. So we return an error instead of falling back to + // older versions of newPayload ENGINE_NEW_PAYLOAD_V3 => get_param::>(params, 0) .map(|jep| JsonExecutionPayload::V3(jep)) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, @@ -337,6 +340,9 @@ pub async fn handle_rpc( } _ => unreachable!(), }), + // From v3 onwards, we use the getPayload version only for the corresponding + // ExecutionPayload version. So we return an error if the ExecutionPayload version + // we get does not correspond to the getPayload version. ENGINE_GET_PAYLOAD_V3 => Ok(match JsonExecutionPayload::from(response) { JsonExecutionPayload::V3(execution_payload) => { serde_json::to_value(JsonGetPayloadResponseV3 { From c8fca4f1d0312392d4ee3e163877b18970d27c69 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 8 May 2024 22:18:07 -0400 Subject: [PATCH 033/227] udpates for aggregate attestation endpoint --- beacon_node/http_api/src/lib.rs | 18 +++++- beacon_node/http_api/tests/tests.rs | 64 ++++++++++++++------- common/eth2/src/lib.rs | 30 +++++++++- common/eth2/src/types.rs | 2 + validator_client/src/attestation_service.rs | 58 +++++++++++++------ 5 files changed, 130 insertions(+), 42 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 6cb8f6fe0b9..fa51c3e3541 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -256,6 +256,7 @@ pub fn prometheus_metrics() -> warp::filters::log::Log( ); // GET validator/aggregate_attestation?attestation_data_root,slot - let get_validator_aggregate_attestation = eth_v1 + let get_validator_aggregate_attestation = any_version .and(warp::path("validator")) .and(warp::path("aggregate_attestation")) .and(warp::path::end()) @@ -3184,11 +3185,24 @@ pub fn serve( .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( - |query: api_types::ValidatorAggregateAttestationQuery, + |endpoint_version: EndpointVersion, + query: api_types::ValidatorAggregateAttestationQuery, not_synced_filter: Result<(), Rejection>, task_spawner: TaskSpawner, chain: Arc>| { task_spawner.blocking_json_task(Priority::P0, move || { + if endpoint_version == V2 { + if query.committee_index.is_none() { + return Err(warp_utils::reject::custom_bad_request( + "missing committee index".to_string(), + )); + } + } else if endpoint_version == V1 { + // Do nothing + } else { + return Err(unsupported_version_rejection(endpoint_version)); + } + //TODO(electra) pass the index into the next method. not_synced_filter?; chain .get_aggregated_attestation_by_slot_and_root( diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 20c22cf12f3..5e22a496a6d 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -3191,30 +3191,52 @@ impl ApiTester { } pub async fn test_get_validator_aggregate_attestation(self) -> Self { - let attestation = self + if self .chain - .head_beacon_block() - .message() - .body() - .attestations() - .next() - .unwrap() - .clone_as_attestation(); - - let result = self - .client - .get_validator_aggregate_attestation( - attestation.data().slot, - attestation.data().tree_hash_root(), - ) - .await - .unwrap() - .unwrap() - .data; + .spec + .fork_name_at_slot::(self.chain.slot().unwrap()) + >= ForkName::Electra + { + for attestation in self.chain.naive_aggregation_pool.read().iter() { + let result = self + .client + .get_validator_aggregate_attestation_v2( + attestation.data().slot, + attestation.data().tree_hash_root(), + attestation.committee_index(), + ) + .await + .unwrap() + .unwrap() + .data; + let expected = attestation; - let expected = attestation; + assert_eq!(&result, expected); + } + } else { + let attestation = self + .chain + .head_beacon_block() + .message() + .body() + .attestations() + .next() + .unwrap() + .clone_as_attestation(); + let result = self + .client + .get_validator_aggregate_attestation_v1( + attestation.data().slot, + attestation.data().tree_hash_root(), + ) + .await + .unwrap() + .unwrap() + .data; + let expected = attestation; - assert_eq!(result, expected); + assert_eq!(result, expected); + } self } diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index d8b2c8ef2d1..532429df3ba 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -2179,8 +2179,8 @@ impl BeaconNodeHttpClient { self.get_with_timeout(path, self.timeouts.attestation).await } - /// `GET validator/aggregate_attestation?slot,attestation_data_root` - pub async fn get_validator_aggregate_attestation( + /// `GET v1/validator/aggregate_attestation?slot,attestation_data_root` + pub async fn get_validator_aggregate_attestation_v1( &self, slot: Slot, attestation_data_root: Hash256, @@ -2203,6 +2203,32 @@ impl BeaconNodeHttpClient { .await } + /// `GET v2/validator/aggregate_attestation?slot,attestation_data_root,committee_index` + pub async fn get_validator_aggregate_attestation_v2( + &self, + slot: Slot, + attestation_data_root: Hash256, + committee_index: CommitteeIndex, + ) -> Result>>, Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("validator") + .push("aggregate_attestation"); + + path.query_pairs_mut() + .append_pair("slot", &slot.to_string()) + .append_pair( + "attestation_data_root", + &format!("{:?}", attestation_data_root), + ) + .append_pair("committee_index", &committee_index.to_string()); + + self.get_opt_with_timeout(path, self.timeouts.attestation) + .await + } + /// `GET validator/sync_committee_contribution` pub async fn get_validator_sync_committee_contribution( &self, diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index b15246e7fdb..2f6e7fe0e82 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -765,6 +765,8 @@ pub struct ValidatorAttestationDataQuery { pub struct ValidatorAggregateAttestationQuery { pub attestation_data_root: Hash256, pub slot: Slot, + #[serde(skip_serializing_if = "Option::is_none")] + pub committee_index: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 5b7f31867bb..062a9436e95 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -14,6 +14,7 @@ use std::ops::Deref; use std::sync::Arc; use tokio::time::{sleep, sleep_until, Duration, Instant}; use tree_hash::TreeHash; +use types::ForkName; use types::{ attestation::AttestationBase, AggregateSignature, Attestation, AttestationData, BitList, ChainSpec, CommitteeIndex, EthSpec, Slot, @@ -290,17 +291,21 @@ impl AttestationService { // Then download, sign and publish a `SignedAggregateAndProof` for each // validator that is elected to aggregate for this `slot` and // `committee_index`. - self.produce_and_publish_aggregates(&attestation_data, &validator_duties) - .await - .map_err(move |e| { - crit!( - log, - "Error during attestation routine"; - "error" => format!("{:?}", e), - "committee_index" => committee_index, - "slot" => slot.as_u64(), - ) - })?; + self.produce_and_publish_aggregates( + &attestation_data, + committee_index, + &validator_duties, + ) + .await + .map_err(move |e| { + crit!( + log, + "Error during attestation routine"; + "error" => format!("{:?}", e), + "committee_index" => committee_index, + "slot" => slot.as_u64(), + ) + })?; } Ok(()) @@ -493,6 +498,7 @@ impl AttestationService { async fn produce_and_publish_aggregates( &self, attestation_data: &AttestationData, + committee_index: CommitteeIndex, validator_duties: &[DutyAndProof], ) -> Result<(), String> { let log = self.context.log(); @@ -505,6 +511,12 @@ impl AttestationService { return Ok(()); } + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); + let aggregated_attestation = &self .beacon_nodes .first_success( @@ -515,12 +527,24 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::AGGREGATES_HTTP_GET], ); - beacon_node - .get_validator_aggregate_attestation( - attestation_data.slot, - attestation_data.tree_hash_root(), - ) - .await + let aggregate_attestation_result = if fork_name >= ForkName::Electra { + beacon_node + .get_validator_aggregate_attestation_v2( + attestation_data.slot, + attestation_data.tree_hash_root(), + committee_index, + ) + .await + } else { + beacon_node + .get_validator_aggregate_attestation_v1( + attestation_data.slot, + attestation_data.tree_hash_root(), + ) + .await + }; + + aggregate_attestation_result .map_err(|e| { format!("Failed to produce an aggregate attestation: {:?}", e) })? From 7cb7653d3614e1455e97cd5a6192671e1c465aa0 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 9 May 2024 17:45:52 +1000 Subject: [PATCH 034/227] Sketch op pool changes --- .../operation_pool/src/attestation_storage.rs | 97 +++++++++++++------ beacon_node/operation_pool/src/lib.rs | 27 ++++-- consensus/types/src/attestation.rs | 3 + 3 files changed, 91 insertions(+), 36 deletions(-) diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index 00fbcbe4b01..5cf93e642c9 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -1,6 +1,6 @@ use crate::AttestationStats; use itertools::Itertools; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use types::{ attestation::{AttestationBase, AttestationElectra}, superstruct, AggregateSignature, Attestation, AttestationData, BeaconState, BitList, BitVector, @@ -42,6 +42,7 @@ pub struct SplitAttestation { pub indexed: CompactIndexedAttestation, } +// TODO(electra): rename this type #[derive(Debug, Clone)] pub struct AttestationRef<'a, E: EthSpec> { pub checkpoint: &'a CheckpointKey, @@ -159,15 +160,15 @@ impl CheckpointKey { } impl CompactIndexedAttestation { - pub fn signers_disjoint_from(&self, other: &Self) -> bool { + pub fn should_aggregate(&self, other: &Self) -> bool { match (self, other) { (CompactIndexedAttestation::Base(this), CompactIndexedAttestation::Base(other)) => { - this.signers_disjoint_from(other) + this.should_aggregate(other) } ( CompactIndexedAttestation::Electra(this), CompactIndexedAttestation::Electra(other), - ) => this.signers_disjoint_from(other), + ) => this.should_aggregate(other), // TODO(electra) is a mix of electra and base compact indexed attestations an edge case we need to deal with? _ => false, } @@ -189,7 +190,7 @@ impl CompactIndexedAttestation { } impl CompactIndexedAttestationBase { - pub fn signers_disjoint_from(&self, other: &Self) -> bool { + pub fn should_aggregate(&self, other: &Self) -> bool { self.aggregation_bits .intersection(&other.aggregation_bits) .is_zero() @@ -208,14 +209,15 @@ impl CompactIndexedAttestationBase { } impl CompactIndexedAttestationElectra { - // TODO(electra) update to match spec requirements - pub fn signers_disjoint_from(&self, other: &Self) -> bool { - self.aggregation_bits - .intersection(&other.aggregation_bits) - .is_zero() + pub fn should_aggregate(&self, other: &Self) -> bool { + // For Electra, only aggregate attestations in the same committee. + self.committee_bits == other.committee_bits + && self + .aggregation_bits + .intersection(&other.aggregation_bits) + .is_zero() } - // TODO(electra) update to match spec requirements pub fn aggregate(&mut self, other: &Self) { self.attesting_indices = self .attesting_indices @@ -226,6 +228,18 @@ impl CompactIndexedAttestationElectra { self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); self.signature.add_assign_aggregate(&other.signature); } + + pub fn committee_index(&self) -> u64 { + *self.get_committee_indices().first().unwrap_or(&0u64) + } + + pub fn get_committee_indices(&self) -> Vec { + self.committee_bits + .iter() + .enumerate() + .filter_map(|(index, bit)| if bit { Some(index as u64) } else { None }) + .collect() + } } impl AttestationMap { @@ -239,34 +253,63 @@ impl AttestationMap { let attestation_map = self.checkpoint_map.entry(checkpoint).or_default(); let attestations = attestation_map.attestations.entry(data).or_default(); - // TODO(electra): // Greedily aggregate the attestation with all existing attestations. // NOTE: this is sub-optimal and in future we will remove this in favour of max-clique // aggregation. let mut aggregated = false; - match attestation { - Attestation::Base(_) => { - for existing_attestation in attestations.iter_mut() { - if existing_attestation.signers_disjoint_from(&indexed) { - existing_attestation.aggregate(&indexed); - aggregated = true; - } else if *existing_attestation == indexed { - aggregated = true; - } - } + for existing_attestation in attestations.iter_mut() { + if existing_attestation.should_aggregate(&indexed) { + existing_attestation.aggregate(&indexed); + aggregated = true; + } else if *existing_attestation == indexed { + aggregated = true; } - // TODO(electra) in order to be devnet ready, we can skip - // aggregating here for now. this will result in "poorly" - // constructed blocks, but that should be fine for devnet - Attestation::Electra(_) => (), - }; + } if !aggregated { attestations.push(indexed); } } + pub fn aggregate_across_committees(&mut self, checkpoint_key: CheckpointKey) { + let Some(attestation_map) = self.checkpoint_map.get_mut(&checkpoint_key) else { + return; + }; + for (compact_attestation_data, compact_indexed_attestations) in + attestation_map.attestations.iter_mut() + { + let unaggregated_attestations = std::mem::take(compact_indexed_attestations); + let mut aggregated_attestations = vec![]; + + // Aggregate the best attestations for each committee and leave the rest. + let mut best_attestations_by_committee = BTreeMap::new(); + + for committee_attestation in unaggregated_attestations { + // TODO(electra) + // compare to best attestations by committee + // could probably use `.entry` here + if let Some(existing_attestation) = + best_attestations_by_committee.get_mut(committee_attestation.committee_index()) + { + // compare and swap, put the discarded one straight into + // `aggregated_attestations` in case we have room to pack it without + // cross-committee aggregation + } else { + best_attestations_by_committee.insert( + committee_attestation.committee_index(), + committee_attestation, + ); + } + } + + // TODO(electra): aggregate all the best attestations by committee + // (use btreemap sort order to get order by committee index) + + *compact_indexed_attestations = aggregated_attestations; + } + } + /// Iterate all attestations matching the given `checkpoint_key`. pub fn get_attestations<'a>( &'a self, diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 6645416d4b0..1d83ecd4651 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -40,7 +40,7 @@ use std::ptr; use types::{ sync_aggregate::Error as SyncAggregateError, typenum::Unsigned, AbstractExecPayload, Attestation, AttestationData, AttesterSlashing, BeaconState, BeaconStateError, ChainSpec, - Epoch, EthSpec, ProposerSlashing, SignedBeaconBlock, SignedBlsToExecutionChange, + Epoch, EthSpec, ForkName, ProposerSlashing, SignedBeaconBlock, SignedBlsToExecutionChange, SignedVoluntaryExit, Slot, SyncAggregate, SyncCommitteeContribution, Validator, }; @@ -256,6 +256,7 @@ impl OperationPool { curr_epoch_validity_filter: impl for<'a> FnMut(&AttestationRef<'a, E>) -> bool + Send, spec: &ChainSpec, ) -> Result>, OpPoolError> { + let fork_name = state.fork_name_unchecked(); if !matches!(state, BeaconState::Base(_)) { // Epoch cache must be initialized to fetch base reward values in the max cover `score` // function. Currently max cover ignores items on errors. If epoch cache is not @@ -267,7 +268,6 @@ impl OperationPool { // Attestations for the current fork, which may be from the current or previous epoch. let (prev_epoch_key, curr_epoch_key) = CheckpointKey::keys_for_state(state); - let all_attestations = self.attestations.read(); let total_active_balance = state .get_total_active_balance() .map_err(OpPoolError::GetAttestationsTotalBalanceError)?; @@ -284,6 +284,14 @@ impl OperationPool { let mut num_prev_valid = 0_i64; let mut num_curr_valid = 0_i64; + // TODO(electra): Work out how to do this more elegantly. This is a bit of a hack. + let mut all_attestations = self.attestations.write(); + + all_attestations.aggregate_across_committees(prev_epoch_key); + all_attestations.aggregate_across_committees(curr_epoch_key); + + let all_attestations = parking_lot::RwLockWriteGuard::downgrade(all_attestations); + let prev_epoch_att = self .get_valid_attestations_for_epoch( &prev_epoch_key, @@ -307,6 +315,11 @@ impl OperationPool { ) .inspect(|_| num_curr_valid += 1); + let curr_epoch_limit = if fork_name < ForkName::Electra { + E::MaxAttestations::to_usize() + } else { + E::MaxAttestationsElectra::to_usize() + }; let prev_epoch_limit = if let BeaconState::Base(base_state) = state { std::cmp::min( E::MaxPendingAttestations::to_usize() @@ -314,7 +327,7 @@ impl OperationPool { E::MaxAttestations::to_usize(), ) } else { - E::MaxAttestations::to_usize() + curr_epoch_limit }; let (prev_cover, curr_cover) = rayon::join( @@ -329,11 +342,7 @@ impl OperationPool { }, move || { let _timer = metrics::start_timer(&metrics::ATTESTATION_CURR_EPOCH_PACKING_TIME); - maximum_cover( - curr_epoch_att, - E::MaxAttestations::to_usize(), - "curr_epoch_attestations", - ) + maximum_cover(curr_epoch_att, curr_epoch_limit, "curr_epoch_attestations") }, ); @@ -343,7 +352,7 @@ impl OperationPool { Ok(max_cover::merge_solutions( curr_cover, prev_cover, - E::MaxAttestations::to_usize(), + curr_epoch_limit, )) } diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index bcecfde10e4..0f7e8468488 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -247,6 +247,9 @@ impl<'a, E: EthSpec> AttestationRef<'a, E> { impl AttestationElectra { /// Are the aggregation bitfields of these attestations disjoint? + // TODO(electra): check whether the definition from CompactIndexedAttestation::should_aggregate + // is useful where this is used, i.e. only consider attestations disjoint when their committees + // match AND their aggregation bits do not intersect. pub fn signers_disjoint_from(&self, other: &Self) -> bool { self.aggregation_bits .intersection(&other.aggregation_bits) From e32dfcdcad69ab129a42238daae6508faa96ec4e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 9 May 2024 09:34:56 -0400 Subject: [PATCH 035/227] fix get attesting indices (#5742) * fix get attesting indices * better errors * fix compile * only get committee index once --- .../beacon_chain/src/attestation_verification.rs | 5 +++-- .../src/common/get_attesting_indices.rs | 15 +++++++++++---- consensus/types/src/beacon_state.rs | 3 ++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 62e65d5f87a..3d722a534be 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -1309,10 +1309,11 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( attesting_indices_electra::get_indexed_attestation(&committees, att) .map(|attestation| (attestation, committees_per_slot)) .map_err(|e| { - if e == BlockOperationError::BeaconStateError(NoCommitteeFound) { + let index = att.committee_index(); + if e == BlockOperationError::BeaconStateError(NoCommitteeFound(index)) { Error::NoCommitteeForSlotAndIndex { slot: att.data.slot, - index: att.committee_index(), + index, } } else { Error::Invalid(e) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 595cc69f87c..9848840e96d 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -113,11 +113,15 @@ pub mod attesting_indices_electra { .map(|committee| (committee.index, committee)) .collect(); + let committee_count_per_slot = committees.len() as u64; + let mut participant_count = 0; for index in committee_indices { if let Some(&beacon_committee) = committees_map.get(&index) { - if aggregation_bits.len() != beacon_committee.committee.len() { - return Err(BeaconStateError::InvalidBitfield); + // This check is new to the spec's `process_attestation` in Electra. + if index >= committee_count_per_slot { + return Err(BeaconStateError::InvalidCommitteeIndex(index)); } + participant_count.safe_add_assign(beacon_committee.committee.len() as u64)?; let committee_attesters = beacon_committee .committee .iter() @@ -136,10 +140,13 @@ pub mod attesting_indices_electra { committee_offset.safe_add(beacon_committee.committee.len())?; } else { - return Err(Error::NoCommitteeFound); + return Err(Error::NoCommitteeFound(index)); } + } - // TODO(electra) what should we do when theres no committee found for a given index? + // This check is new to the spec's `process_attestation` in Electra. + if participant_count as usize != aggregation_bits.len() { + return Err(BeaconStateError::InvalidBitfield); } let mut indices = output.into_iter().collect_vec(); diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 599c0bfc39c..d9c7a78537a 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -159,7 +159,8 @@ pub enum Error { IndexNotSupported(usize), InvalidFlagIndex(usize), MerkleTreeError(merkle_proof::MerkleTreeError), - NoCommitteeFound, + NoCommitteeFound(CommitteeIndex), + InvalidCommitteeIndex(CommitteeIndex), } /// Control whether an epoch-indexed field can be indexed at the next epoch or not. From f5eb33d5482c95f3f23aa2fad97886a5f07e1f95 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 9 May 2024 11:25:14 -0400 Subject: [PATCH 036/227] add new gossip conditions --- .../src/attestation_verification.rs | 65 ++++++++++++++++--- beacon_node/beacon_chain/src/errors.rs | 2 + .../gossip_methods.rs | 34 ++++++++++ .../src/common/get_attesting_indices.rs | 2 +- consensus/types/src/attestation.rs | 5 +- 5 files changed, 98 insertions(+), 10 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3d722a534be..c9742d5468f 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -45,7 +45,10 @@ use proto_array::Block as ProtoBlock; use slog::debug; use slot_clock::SlotClock; use state_processing::{ - common::{attesting_indices_base, attesting_indices_electra}, + common::{ + attesting_indices_base, + attesting_indices_electra::{self, get_committee_indices}, + }, per_block_processing::errors::{AttestationValidationError, BlockOperationError}, signature_sets::{ indexed_attestation_signature_set_from_pubkeys, @@ -139,6 +142,12 @@ pub enum Error { /// /// The peer has sent an invalid message. ValidatorIndexTooHigh(usize), + /// The validator index is not set to zero after Electra. + /// + /// ## Peer scoring + /// + /// The peer has sent an invalid message. + CommitteeIndexNonZero(usize), /// The `attestation.data.beacon_block_root` block is unknown. /// /// ## Peer scoring @@ -187,6 +196,12 @@ pub enum Error { /// /// The peer has sent an invalid message. NotExactlyOneAggregationBitSet(usize), + /// The attestation doesn't have only one aggregation bit set. + /// + /// ## Peer scoring + /// + /// The peer has sent an invalid message. + NotExactlyOneCommitteeBitSet(usize), /// We have already observed an attestation for the `validator_index` and refuse to process /// another. /// @@ -466,6 +481,9 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { }); } + // [New in Electra:EIP7549] + verify_committee_index(attestation, &chain.spec)?; + // Ensure the valid aggregated attestation has not already been seen locally. let attestation_data = attestation.data(); let attestation_data_root = attestation_data.tree_hash_root(); @@ -775,6 +793,9 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { return Err(Error::NotExactlyOneAggregationBitSet(num_aggregation_bits)); } + // [New in Electra:EIP7549] + verify_committee_index(attestation, &chain.spec)?; + // Attestations must be for a known block. If the block is unknown, we simply drop the // attestation and do not delay consideration for later. // @@ -1101,14 +1122,13 @@ pub fn verify_propagation_slot_range( let current_fork = spec.fork_name_at_slot::(slot_clock.now().ok_or(BeaconChainError::UnableToReadSlot)?); - let earliest_permissible_slot = match current_fork { - ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { - one_epoch_prior - } - // EIP-7045 - ForkName::Deneb | ForkName::Electra => one_epoch_prior + let earliest_permissible_slot = if current_fork < ForkName::Deneb { + one_epoch_prior + // EIP-7045 + } else { + one_epoch_prior .epoch(E::slots_per_epoch()) - .start_slot(E::slots_per_epoch()), + .start_slot(E::slots_per_epoch()) }; if attestation_slot < earliest_permissible_slot { @@ -1273,6 +1293,35 @@ pub fn verify_signed_aggregate_signatures( Ok(verify_signature_sets(signature_sets.iter())) } +/// Verify that the `attestation` committee index is properly set for the attestation's fork. +/// This function will only apply verification post-Electra. +pub fn verify_committee_index( + attestation: AttestationRef, + spec: &ChainSpec, +) -> Result<(), Error> { + if spec.fork_name_at_slot::(attestation.data().slot) >= ForkName::Electra { + // Check to ensure that the attestation is for a single committee. + let num_committee_bits = get_committee_indices::( + attestation + .committee_bits() + .map_err(|e| Error::BeaconChainError(e.into()))?, + ); + if num_committee_bits.len() != 1 { + return Err(Error::NotExactlyOneCommitteeBitSet( + num_committee_bits.len(), + )); + } + + // Ensure the attestation index is set to zero post Electra. + if attestation.data().index != 0 { + return Err(Error::CommitteeIndexNonZero( + attestation.data().index as usize, + )); + } + } + Ok(()) +} + /// Assists in readability. type CommitteesPerSlot = u64; diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 340f1f9f797..b210536a628 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -226,6 +226,7 @@ pub enum BeaconChainError { LightClientError(LightClientError), UnsupportedFork, MilhouseError(MilhouseError), + AttestationError(AttestationError), } easy_from_to!(SlotProcessingError, BeaconChainError); @@ -256,6 +257,7 @@ easy_from_to!(AvailabilityCheckError, BeaconChainError); easy_from_to!(EpochCacheError, BeaconChainError); easy_from_to!(LightClientError, BeaconChainError); easy_from_to!(MilhouseError, BeaconChainError); +easy_from_to!(AttestationError, BeaconChainError); #[derive(Debug)] pub enum BlockProductionError { diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index d41233c903b..d1e7f54abf0 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -2038,6 +2038,27 @@ impl NetworkBeaconProcessor { "attn_val_index_too_high", ); } + AttnError::CommitteeIndexNonZero(index) => { + /* + * The validator index is not set to zero after Electra. + * + * The peer has published an invalid consensus message. + */ + debug!( + self.log, + "Committee index non zero"; + "peer_id" => %peer_id, + "block" => ?beacon_block_root, + "type" => ?attestation_type, + "committee_index" => index, + ); + self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Reject); + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "attn_comm_index_non_zero", + ); + } AttnError::UnknownHeadBlock { beacon_block_root } => { trace!( self.log, @@ -2193,6 +2214,19 @@ impl NetworkBeaconProcessor { "attn_too_many_agg_bits", ); } + AttnError::NotExactlyOneCommitteeBitSet(_) => { + /* + * The attestation doesn't have only one committee bit set. + * + * The peer has published an invalid consensus message. + */ + self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Reject); + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "attn_too_many_comm_bits", + ); + } AttnError::AttestsToFutureBlock { .. } => { /* * The beacon_block_root is from a higher slot than the attestation. diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 9848840e96d..1c59edf20e7 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -155,7 +155,7 @@ pub mod attesting_indices_electra { Ok(indices) } - fn get_committee_indices( + pub fn get_committee_indices( committee_bits: &BitVector, ) -> Vec { committee_bits diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index bcecfde10e4..8f836810a7e 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -22,6 +22,7 @@ pub enum Error { SszTypesError(ssz_types::Error), AlreadySigned(usize), SubnetCountIsZero(ArithError), + IncorrectStateVariant, } #[superstruct( @@ -43,7 +44,9 @@ pub enum Error { serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), ), - ref_attributes(derive(TreeHash), tree_hash(enum_behaviour = "transparent")) + ref_attributes(derive(TreeHash), tree_hash(enum_behaviour = "transparent")), + cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"), + partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant") )] #[derive( Debug, From 07229b76ed89dadd6ce7464699ed5f9a265e8851 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 9 May 2024 13:40:52 -0400 Subject: [PATCH 037/227] Ef test fixes (#5753) * attestation related ef test fixes * delete commented out stuff --- testing/ef_tests/src/type_name.rs | 7 ++--- testing/ef_tests/tests/tests.rs | 49 +++++++++++++++++++------------ 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 30db5c0e4a9..cbea78dabfc 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -111,11 +111,8 @@ type_name_generic!(LightClientUpdateDeneb, "LightClientUpdate"); type_name_generic!(PendingAttestation); type_name!(ProposerSlashing); type_name_generic!(SignedAggregateAndProof); -type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProofBase"); -type_name_generic!( - SignedAggregateAndProofElectra, - "SignedAggregateAndProofElectra" -); +type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProof"); +type_name_generic!(SignedAggregateAndProofElectra, "SignedAggregateAndProof"); type_name_generic!(SignedBeaconBlock); type_name!(SignedBeaconBlockHeader); type_name_generic!(SignedContributionAndProof); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 85d9362aaeb..fb8bdfcae71 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -219,7 +219,6 @@ mod ssz_static { use types::historical_summary::HistoricalSummary; use types::{AttesterSlashingBase, AttesterSlashingElectra, LightClientBootstrapAltair, *}; - ssz_static_test!(aggregate_and_proof, AggregateAndProof<_>); ssz_static_test!(attestation, Attestation<_>); ssz_static_test!(attestation_data, AttestationData); ssz_static_test!(beacon_block, SszStaticWithSpecHandler, BeaconBlock<_>); @@ -249,7 +248,7 @@ mod ssz_static { ssz_static_test!(voluntary_exit, VoluntaryExit); #[test] - fn signed_aggregate_and_proof() { + fn attester_slashing() { SszStaticHandler::, MinimalEthSpec>::pre_electra() .run(); SszStaticHandler::, MainnetEthSpec>::pre_electra() @@ -260,6 +259,36 @@ mod ssz_static { .run(); } + #[test] + fn signed_aggregate_and_proof() { + SszStaticHandler::, MinimalEthSpec>::pre_electra( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::pre_electra( + ) + .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); + } + + #[test] + fn aggregate_and_proof() { + SszStaticHandler::, MinimalEthSpec>::pre_electra() + .run(); + SszStaticHandler::, MainnetEthSpec>::pre_electra() + .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); + } + // BeaconBlockBody has no internal indicator of which fork it is for, so we test it separately. #[test] fn beacon_block_body() { @@ -283,22 +312,6 @@ mod ssz_static { .run(); } - #[test] - fn signed_aggregate_and_proof() { - SszStaticHandler::, MinimalEthSpec>::pre_electra( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::pre_electra( - ) - .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( - ) - .run(); - } - // Altair and later #[test] fn contribution_and_proof() { From cb8c8f59cf6dc21e7804f45679f4d3aad5f319ee Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Thu, 9 May 2024 14:50:11 -0500 Subject: [PATCH 038/227] Fix Aggregation Pool for Electra (#5754) * Fix Aggregation Pool for Electra * Remove Outdated Interface --- beacon_node/beacon_chain/src/beacon_chain.rs | 38 ++- .../src/naive_aggregation_pool.rs | 282 ++++++++++++++---- beacon_node/beacon_chain/src/test_utils.rs | 2 +- .../tests/payload_invalidation.rs | 5 +- beacon_node/http_api/src/lib.rs | 2 +- consensus/types/src/attestation.rs | 1 + 6 files changed, 255 insertions(+), 75 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 973dcaadb47..331e04069fd 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1612,11 +1612,28 @@ impl BeaconChain { /// Returns an aggregated `Attestation`, if any, that has a matching `attestation.data`. /// /// The attestation will be obtained from `self.naive_aggregation_pool`. - pub fn get_aggregated_attestation( + pub fn get_aggregated_attestation_base( &self, data: &AttestationData, ) -> Result>, Error> { - if let Some(attestation) = self.naive_aggregation_pool.read().get(data) { + let attestation_key = crate::naive_aggregation_pool::AttestationKey::new_base(data); + if let Some(attestation) = self.naive_aggregation_pool.read().get(&attestation_key) { + self.filter_optimistic_attestation(attestation) + .map(Option::Some) + } else { + Ok(None) + } + } + + // TODO(electra): call this function from the new beacon API method + pub fn get_aggregated_attestation_electra( + &self, + data: &AttestationData, + committee_index: CommitteeIndex, + ) -> Result>, Error> { + let attestation_key = + crate::naive_aggregation_pool::AttestationKey::new_electra(data, committee_index); + if let Some(attestation) = self.naive_aggregation_pool.read().get(&attestation_key) { self.filter_optimistic_attestation(attestation) .map(Option::Some) } else { @@ -1628,16 +1645,21 @@ impl BeaconChain { /// `attestation.data.tree_hash_root()`. /// /// The attestation will be obtained from `self.naive_aggregation_pool`. - pub fn get_aggregated_attestation_by_slot_and_root( + /// + /// NOTE: This function will *only* work with pre-electra attestations and it only + /// exists to support the pre-electra validator API method. + pub fn get_pre_electra_aggregated_attestation_by_slot_and_root( &self, slot: Slot, attestation_data_root: &Hash256, ) -> Result>, Error> { - if let Some(attestation) = self - .naive_aggregation_pool - .read() - .get_by_slot_and_root(slot, attestation_data_root) - { + let attestation_key = + crate::naive_aggregation_pool::AttestationKey::new_base_from_slot_and_root( + slot, + *attestation_data_root, + ); + + if let Some(attestation) = self.naive_aggregation_pool.read().get(&attestation_key) { self.filter_optimistic_attestation(attestation) .map(Option::Some) } else { diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index a12521cd171..6c4f7cdae72 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -1,7 +1,9 @@ use crate::metrics; use crate::observed_aggregates::AsReference; +use itertools::Itertools; +use smallvec::SmallVec; use std::collections::HashMap; -use tree_hash::TreeHash; +use tree_hash::{MerkleHasher, TreeHash, TreeHashType}; use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT; use types::slot_data::SlotData; use types::sync_committee_contribution::SyncContributionData; @@ -9,9 +11,114 @@ use types::{ Attestation, AttestationData, AttestationRef, EthSpec, Hash256, Slot, SyncCommitteeContribution, }; -type AttestationDataRoot = Hash256; +type AttestationKeyRoot = Hash256; type SyncDataRoot = Hash256; +/// Post-Electra, we need a new key for Attestations that includes the committee index +#[derive(Debug, Clone, PartialEq)] +pub struct AttestationKey { + data_root: Hash256, + committee_index: Option, + slot: Slot, +} + +// A custom implementation of `TreeHash` such that: +// AttestationKey(data, None).tree_hash_root() == data.tree_hash_root() +// AttestationKey(data, Some(index)).tree_hash_root() == (data, index).tree_hash_root() +// This is necessary because pre-Electra, the validator will ask for the tree_hash_root() +// of the `AttestationData` +impl TreeHash for AttestationKey { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Container + } + + fn tree_hash_packed_encoding(&self) -> SmallVec<[u8; 32]> { + unreachable!("AttestationKey should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("AttestationKey should never be packed.") + } + + fn tree_hash_root(&self) -> Hash256 { + match self.committee_index { + None => self.data_root, // Return just the data root if no committee index is present + Some(index) => { + // Combine the hash of the data with the hash of the index + let mut hasher = MerkleHasher::with_leaves(2); + hasher + .write(self.data_root.as_bytes()) + .expect("should write data hash"); + hasher + .write(&index.to_le_bytes()) + .expect("should write index"); + hasher.finish().expect("should give tree hash") + } + } + } +} + +impl AttestationKey { + pub fn from_attestation_ref(attestation: AttestationRef) -> Result { + let slot = attestation.data().slot; + match attestation { + AttestationRef::Base(att) => Ok(Self { + data_root: att.data.tree_hash_root(), + committee_index: None, + slot, + }), + AttestationRef::Electra(att) => { + let committee_index = att + .committee_bits + .iter() + .enumerate() + .filter_map(|(i, bit)| if bit { Some(i) } else { None }) + .at_most_one() + .map_err(|_| Error::MoreThanOneCommitteeBitSet)? + .ok_or(Error::NoCommitteeBitSet)?; + + Ok(Self { + data_root: att.data.tree_hash_root(), + committee_index: Some(committee_index as u64), + slot, + }) + } + } + } + + pub fn new_base(data: &AttestationData) -> Self { + let slot = data.slot; + Self { + data_root: data.tree_hash_root(), + committee_index: None, + slot, + } + } + + pub fn new_electra(data: &AttestationData, committee_index: u64) -> Self { + let slot = data.slot; + Self { + data_root: data.tree_hash_root(), + committee_index: Some(committee_index), + slot, + } + } + + pub fn new_base_from_slot_and_root(slot: Slot, data_root: Hash256) -> Self { + Self { + data_root, + committee_index: None, + slot, + } + } +} + +impl SlotData for AttestationKey { + fn get_slot(&self) -> Slot { + self.slot + } +} + /// The number of slots that will be stored in the pool. /// /// For example, if `SLOTS_RETAINED == 3` and the pool is pruned at slot `6`, then all items @@ -49,6 +156,10 @@ pub enum Error { /// The given `aggregation_bits` field had more than one signature. The number of /// signatures found is included. MoreThanOneAggregationBitSet(usize), + /// The electra attestation has more than one committee bit set + MoreThanOneCommitteeBitSet, + /// The electra attestation has NO committee bit set + NoCommitteeBitSet, /// We have reached the maximum number of unique items that can be stored in a /// slot. This is a DoS protection function. ReachedMaxItemsPerSlot(usize), @@ -90,9 +201,6 @@ where /// Get a reference to the inner `HashMap`. fn get_map(&self) -> &HashMap; - /// Get a `Value` from `Self` based on `Key`, which is a hash of `Data`. - fn get_by_root(&self, root: &Self::Key) -> Option<&Self::Value>; - /// The number of items store in `Self`. fn len(&self) -> usize; @@ -112,13 +220,13 @@ where /// A collection of `Attestation` objects, keyed by their `attestation.data`. Enforces that all /// `attestation` are from the same slot. pub struct AggregatedAttestationMap { - map: HashMap>, + map: HashMap>, } impl AggregateMap for AggregatedAttestationMap { - type Key = AttestationDataRoot; + type Key = AttestationKeyRoot; type Value = Attestation; - type Data = AttestationData; + type Data = AttestationKey; /// Create an empty collection with the given `initial_capacity`. fn new(initial_capacity: usize) -> Self { @@ -133,45 +241,43 @@ impl AggregateMap for AggregatedAttestationMap { fn insert(&mut self, a: AttestationRef) -> Result { let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_CORE_INSERT); - let set_bits = match a { + let aggregation_bit = match a { AttestationRef::Base(att) => att .aggregation_bits .iter() .enumerate() - .filter(|(_i, bit)| *bit) - .map(|(i, _bit)| i) - .collect::>(), + .filter_map(|(i, bit)| if bit { Some(i) } else { None }) + .at_most_one() + .map_err(|iter| Error::MoreThanOneAggregationBitSet(iter.count()))? + .ok_or(Error::NoAggregationBitsSet)?, AttestationRef::Electra(att) => att .aggregation_bits .iter() .enumerate() - .filter(|(_i, bit)| *bit) - .map(|(i, _bit)| i) - .collect::>(), + .filter_map(|(i, bit)| if bit { Some(i) } else { None }) + .at_most_one() + .map_err(|iter| Error::MoreThanOneAggregationBitSet(iter.count()))? + .ok_or(Error::NoAggregationBitsSet)?, }; - let committee_index = set_bits - .first() - .copied() - .ok_or(Error::NoAggregationBitsSet)?; - - if set_bits.len() > 1 { - return Err(Error::MoreThanOneAggregationBitSet(set_bits.len())); - } - - let attestation_data_root = a.data().tree_hash_root(); + let attestation_key = AttestationKey::from_attestation_ref(a)?; + let attestation_data_root = attestation_key.tree_hash_root(); if let Some(existing_attestation) = self.map.get_mut(&attestation_data_root) { if existing_attestation - .get_aggregation_bit(committee_index) + .get_aggregation_bit(aggregation_bit) .map_err(|_| Error::InconsistentBitfieldLengths)? { - Ok(InsertOutcome::SignatureAlreadyKnown { committee_index }) + Ok(InsertOutcome::SignatureAlreadyKnown { + committee_index: aggregation_bit, + }) } else { let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_AGGREGATION); existing_attestation.aggregate(a); - Ok(InsertOutcome::SignatureAggregated { committee_index }) + Ok(InsertOutcome::SignatureAggregated { + committee_index: aggregation_bit, + }) } } else { if self.map.len() >= MAX_ATTESTATIONS_PER_SLOT { @@ -180,7 +286,9 @@ impl AggregateMap for AggregatedAttestationMap { self.map .insert(attestation_data_root, a.clone_as_attestation()); - Ok(InsertOutcome::NewItemInserted { committee_index }) + Ok(InsertOutcome::NewItemInserted { + committee_index: aggregation_bit, + }) } } @@ -195,11 +303,6 @@ impl AggregateMap for AggregatedAttestationMap { &self.map } - /// Returns an aggregated `Attestation` with the given `root`, if any. - fn get_by_root(&self, root: &Self::Key) -> Option<&Self::Value> { - self.map.get(root) - } - fn len(&self) -> usize { self.map.len() } @@ -306,11 +409,6 @@ impl AggregateMap for SyncContributionAggregateMap { &self.map } - /// Returns an aggregated `SyncCommitteeContribution` with the given `root`, if any. - fn get_by_root(&self, root: &SyncDataRoot) -> Option<&SyncCommitteeContribution> { - self.map.get(root) - } - fn len(&self) -> usize { self.map.len() } @@ -445,13 +543,6 @@ where .and_then(|map| map.get(data)) } - /// Returns an aggregated `T::Value` with the given `slot` and `root`, if any. - pub fn get_by_slot_and_root(&self, slot: Slot, root: &T::Key) -> Option { - self.maps - .get(&slot) - .and_then(|map| map.get_by_root(root).cloned()) - } - /// Iterate all items in all slots of `self`. pub fn iter(&self) -> impl Iterator { self.maps.values().flat_map(|map| map.get_map().values()) @@ -500,19 +591,30 @@ mod tests { use super::*; use ssz_types::BitList; use store::BitVector; + use tree_hash::TreeHash; use types::{ test_utils::{generate_deterministic_keypair, test_random_instance}, - Fork, Hash256, SyncCommitteeMessage, + Attestation, AttestationBase, AttestationElectra, Fork, Hash256, SyncCommitteeMessage, }; type E = types::MainnetEthSpec; - fn get_attestation(slot: Slot) -> Attestation { - let mut a: Attestation = test_random_instance(); - a.data_mut().slot = slot; - *a.aggregation_bits_base_mut().unwrap() = - BitList::with_capacity(4).expect("should create bitlist"); - a + fn get_attestation_base(slot: Slot) -> Attestation { + let mut a: AttestationBase = test_random_instance(); + a.data.slot = slot; + a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); + Attestation::Base(a) + } + + fn get_attestation_electra(slot: Slot) -> Attestation { + let mut a: AttestationElectra = test_random_instance(); + a.data.slot = slot; + a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); + a.committee_bits = BitVector::new(); + a.committee_bits + .set(0, true) + .expect("should set committee bit"); + Attestation::Electra(a) } fn get_sync_contribution(slot: Slot) -> SyncCommitteeContribution { @@ -555,10 +657,16 @@ mod tests { } fn unset_attestation_bit(a: &mut Attestation, i: usize) { - a.aggregation_bits_base_mut() - .unwrap() - .set(i, false) - .expect("should unset aggregation bit") + match a { + Attestation::Base(ref mut att) => att + .aggregation_bits + .set(i, false) + .expect("should unset aggregation bit"), + Attestation::Electra(ref mut att) => att + .aggregation_bits + .set(i, false) + .expect("should unset aggregation bit"), + } } fn unset_sync_contribution_bit(a: &mut SyncCommitteeContribution, i: usize) { @@ -579,8 +687,8 @@ mod tests { a.data().beacon_block_root == block_root } - fn key_from_attestation(a: &Attestation) -> AttestationData { - a.data().clone() + fn key_from_attestation(a: &Attestation) -> AttestationKey { + AttestationKey::from_attestation_ref(a.to_ref()).expect("should create attestation key") } fn mutate_sync_contribution_block_root( @@ -605,6 +713,45 @@ mod tests { SyncContributionData::from_contribution(a) } + #[test] + fn attestation_key_tree_hash_tests() { + let attestation_base = get_attestation_base(Slot::new(42)); + // for a base attestation, the tree_hash_root() of the key should be the same as the tree_hash_root() of the data + let attestation_key_base = AttestationKey::from_attestation_ref(attestation_base.to_ref()) + .expect("should create attestation key"); + assert_eq!( + attestation_key_base.tree_hash_root(), + attestation_base.data().tree_hash_root() + ); + let mut attestation_electra = get_attestation_electra(Slot::new(42)); + // for an electra attestation, the tree_hash_root() of the key should be different from the tree_hash_root() of the data + let attestation_key_electra = + AttestationKey::from_attestation_ref(attestation_electra.to_ref()) + .expect("should create attestation key"); + assert_ne!( + attestation_key_electra.tree_hash_root(), + attestation_electra.data().tree_hash_root() + ); + // for an electra attestation, the tree_hash_root() of the key should be dependent on which committee bit is set + let committe_bits = attestation_electra + .committee_bits_mut() + .expect("should get committee bits"); + committe_bits + .set(0, false) + .expect("should set committee bit"); + committe_bits + .set(1, true) + .expect("should set committee bit"); + let new_attestation_key_electra = + AttestationKey::from_attestation_ref(attestation_electra.to_ref()) + .expect("should create attestation key"); + // this new key should have a different tree_hash_root() than the previous key + assert_ne!( + attestation_key_electra.tree_hash_root(), + new_attestation_key_electra.tree_hash_root() + ); + } + macro_rules! test_suite { ( $mod_name: ident, @@ -800,8 +947,21 @@ mod tests { } test_suite! { - attestation_tests, - get_attestation, + attestation_tests_base, + get_attestation_base, + sign_attestation, + unset_attestation_bit, + mutate_attestation_block_root, + mutate_attestation_slot, + attestation_block_root_comparator, + key_from_attestation, + AggregatedAttestationMap, + MAX_ATTESTATIONS_PER_SLOT + } + + test_suite! { + attestation_tests_electra, + get_attestation_electra, sign_attestation, unset_attestation_bit, mutate_attestation_block_root, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index edfff4bf81e..bcf7582ebfd 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1374,7 +1374,7 @@ where // aggregate locally. let aggregate = self .chain - .get_aggregated_attestation(attestation.data()) + .get_aggregated_attestation_base(attestation.data()) .unwrap() .unwrap_or_else(|| { committee_attestations.iter().skip(1).fold( diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 8c9957db169..594872e2fff 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -1228,10 +1228,7 @@ async fn attesting_to_optimistic_head() { let get_aggregated_by_slot_and_root = || { rig.harness .chain - .get_aggregated_attestation_by_slot_and_root( - attestation.data().slot, - &attestation.data().tree_hash_root(), - ) + .get_aggregated_attestation_base(attestation.data()) }; /* diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 6cb8f6fe0b9..838f7233052 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3191,7 +3191,7 @@ pub fn serve( task_spawner.blocking_json_task(Priority::P0, move || { not_synced_filter?; chain - .get_aggregated_attestation_by_slot_and_root( + .get_pre_electra_aggregated_attestation_by_slot_and_root( query.slot, &query.attestation_data_root, ) diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index bcecfde10e4..6d5cb5bd1a9 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -92,6 +92,7 @@ impl Decode for Attestation { } } +// TODO(electra): think about how to handle fork variants here impl TestRandom for Attestation { fn random_for_test(rng: &mut impl RngCore) -> Self { let aggregation_bits: BitList = BitList::random_for_test(rng); From c30f70906be1eaeb62d8374fc7d1b06af06fd109 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 9 May 2024 17:49:12 -0400 Subject: [PATCH 039/227] fix ssz (#5755) --- consensus/types/src/attestation.rs | 2 +- consensus/types/src/eth_spec.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 6d5cb5bd1a9..9dd4f1c9241 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -67,9 +67,9 @@ pub struct Attestation { #[superstruct(only(Electra), partial_getter(rename = "aggregation_bits_electra"))] pub aggregation_bits: BitList, pub data: AttestationData, - pub signature: AggregateSignature, #[superstruct(only(Electra))] pub committee_bits: BitVector, + pub signature: AggregateSignature, } impl Decode for Attestation { diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index cec4db2da51..14949e67531 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -408,6 +408,8 @@ impl EthSpec for MainnetEthSpec { pub struct MinimalEthSpec; impl EthSpec for MinimalEthSpec { + type MaxCommitteesPerSlot = U4; + type MaxValidatorsPerSlot = U8192; type SlotsPerEpoch = U8; type EpochsPerEth1VotingPeriod = U4; type SlotsPerHistoricalRoot = U64; @@ -432,8 +434,6 @@ impl EthSpec for MinimalEthSpec { SubnetBitfieldLength, SyncCommitteeSubnetCount, MaxValidatorsPerCommittee, - MaxCommitteesPerSlot, - MaxValidatorsPerSlot, GenesisEpoch, HistoricalRootsLimit, ValidatorRegistryLimit, From ab9e58aa3d0e6fe2175a4996a5de710e81152896 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Thu, 9 May 2024 16:59:39 -0500 Subject: [PATCH 040/227] Get `electra_op_pool` up to date (#5756) * fix get attesting indices (#5742) * fix get attesting indices * better errors * fix compile * only get committee index once * Ef test fixes (#5753) * attestation related ef test fixes * delete commented out stuff * Fix Aggregation Pool for Electra (#5754) * Fix Aggregation Pool for Electra * Remove Outdated Interface * fix ssz (#5755) --------- Co-authored-by: realbigsean --- .../src/attestation_verification.rs | 5 +- beacon_node/beacon_chain/src/beacon_chain.rs | 38 ++- .../src/naive_aggregation_pool.rs | 282 ++++++++++++++---- beacon_node/beacon_chain/src/test_utils.rs | 2 +- .../tests/payload_invalidation.rs | 5 +- beacon_node/http_api/src/lib.rs | 2 +- .../src/common/get_attesting_indices.rs | 15 +- consensus/types/src/attestation.rs | 3 +- consensus/types/src/beacon_state.rs | 3 +- consensus/types/src/eth_spec.rs | 4 +- testing/ef_tests/src/type_name.rs | 7 +- testing/ef_tests/tests/tests.rs | 49 +-- 12 files changed, 307 insertions(+), 108 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 62e65d5f87a..3d722a534be 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -1309,10 +1309,11 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( attesting_indices_electra::get_indexed_attestation(&committees, att) .map(|attestation| (attestation, committees_per_slot)) .map_err(|e| { - if e == BlockOperationError::BeaconStateError(NoCommitteeFound) { + let index = att.committee_index(); + if e == BlockOperationError::BeaconStateError(NoCommitteeFound(index)) { Error::NoCommitteeForSlotAndIndex { slot: att.data.slot, - index: att.committee_index(), + index, } } else { Error::Invalid(e) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 973dcaadb47..331e04069fd 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1612,11 +1612,28 @@ impl BeaconChain { /// Returns an aggregated `Attestation`, if any, that has a matching `attestation.data`. /// /// The attestation will be obtained from `self.naive_aggregation_pool`. - pub fn get_aggregated_attestation( + pub fn get_aggregated_attestation_base( &self, data: &AttestationData, ) -> Result>, Error> { - if let Some(attestation) = self.naive_aggregation_pool.read().get(data) { + let attestation_key = crate::naive_aggregation_pool::AttestationKey::new_base(data); + if let Some(attestation) = self.naive_aggregation_pool.read().get(&attestation_key) { + self.filter_optimistic_attestation(attestation) + .map(Option::Some) + } else { + Ok(None) + } + } + + // TODO(electra): call this function from the new beacon API method + pub fn get_aggregated_attestation_electra( + &self, + data: &AttestationData, + committee_index: CommitteeIndex, + ) -> Result>, Error> { + let attestation_key = + crate::naive_aggregation_pool::AttestationKey::new_electra(data, committee_index); + if let Some(attestation) = self.naive_aggregation_pool.read().get(&attestation_key) { self.filter_optimistic_attestation(attestation) .map(Option::Some) } else { @@ -1628,16 +1645,21 @@ impl BeaconChain { /// `attestation.data.tree_hash_root()`. /// /// The attestation will be obtained from `self.naive_aggregation_pool`. - pub fn get_aggregated_attestation_by_slot_and_root( + /// + /// NOTE: This function will *only* work with pre-electra attestations and it only + /// exists to support the pre-electra validator API method. + pub fn get_pre_electra_aggregated_attestation_by_slot_and_root( &self, slot: Slot, attestation_data_root: &Hash256, ) -> Result>, Error> { - if let Some(attestation) = self - .naive_aggregation_pool - .read() - .get_by_slot_and_root(slot, attestation_data_root) - { + let attestation_key = + crate::naive_aggregation_pool::AttestationKey::new_base_from_slot_and_root( + slot, + *attestation_data_root, + ); + + if let Some(attestation) = self.naive_aggregation_pool.read().get(&attestation_key) { self.filter_optimistic_attestation(attestation) .map(Option::Some) } else { diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index a12521cd171..6c4f7cdae72 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -1,7 +1,9 @@ use crate::metrics; use crate::observed_aggregates::AsReference; +use itertools::Itertools; +use smallvec::SmallVec; use std::collections::HashMap; -use tree_hash::TreeHash; +use tree_hash::{MerkleHasher, TreeHash, TreeHashType}; use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT; use types::slot_data::SlotData; use types::sync_committee_contribution::SyncContributionData; @@ -9,9 +11,114 @@ use types::{ Attestation, AttestationData, AttestationRef, EthSpec, Hash256, Slot, SyncCommitteeContribution, }; -type AttestationDataRoot = Hash256; +type AttestationKeyRoot = Hash256; type SyncDataRoot = Hash256; +/// Post-Electra, we need a new key for Attestations that includes the committee index +#[derive(Debug, Clone, PartialEq)] +pub struct AttestationKey { + data_root: Hash256, + committee_index: Option, + slot: Slot, +} + +// A custom implementation of `TreeHash` such that: +// AttestationKey(data, None).tree_hash_root() == data.tree_hash_root() +// AttestationKey(data, Some(index)).tree_hash_root() == (data, index).tree_hash_root() +// This is necessary because pre-Electra, the validator will ask for the tree_hash_root() +// of the `AttestationData` +impl TreeHash for AttestationKey { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Container + } + + fn tree_hash_packed_encoding(&self) -> SmallVec<[u8; 32]> { + unreachable!("AttestationKey should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("AttestationKey should never be packed.") + } + + fn tree_hash_root(&self) -> Hash256 { + match self.committee_index { + None => self.data_root, // Return just the data root if no committee index is present + Some(index) => { + // Combine the hash of the data with the hash of the index + let mut hasher = MerkleHasher::with_leaves(2); + hasher + .write(self.data_root.as_bytes()) + .expect("should write data hash"); + hasher + .write(&index.to_le_bytes()) + .expect("should write index"); + hasher.finish().expect("should give tree hash") + } + } + } +} + +impl AttestationKey { + pub fn from_attestation_ref(attestation: AttestationRef) -> Result { + let slot = attestation.data().slot; + match attestation { + AttestationRef::Base(att) => Ok(Self { + data_root: att.data.tree_hash_root(), + committee_index: None, + slot, + }), + AttestationRef::Electra(att) => { + let committee_index = att + .committee_bits + .iter() + .enumerate() + .filter_map(|(i, bit)| if bit { Some(i) } else { None }) + .at_most_one() + .map_err(|_| Error::MoreThanOneCommitteeBitSet)? + .ok_or(Error::NoCommitteeBitSet)?; + + Ok(Self { + data_root: att.data.tree_hash_root(), + committee_index: Some(committee_index as u64), + slot, + }) + } + } + } + + pub fn new_base(data: &AttestationData) -> Self { + let slot = data.slot; + Self { + data_root: data.tree_hash_root(), + committee_index: None, + slot, + } + } + + pub fn new_electra(data: &AttestationData, committee_index: u64) -> Self { + let slot = data.slot; + Self { + data_root: data.tree_hash_root(), + committee_index: Some(committee_index), + slot, + } + } + + pub fn new_base_from_slot_and_root(slot: Slot, data_root: Hash256) -> Self { + Self { + data_root, + committee_index: None, + slot, + } + } +} + +impl SlotData for AttestationKey { + fn get_slot(&self) -> Slot { + self.slot + } +} + /// The number of slots that will be stored in the pool. /// /// For example, if `SLOTS_RETAINED == 3` and the pool is pruned at slot `6`, then all items @@ -49,6 +156,10 @@ pub enum Error { /// The given `aggregation_bits` field had more than one signature. The number of /// signatures found is included. MoreThanOneAggregationBitSet(usize), + /// The electra attestation has more than one committee bit set + MoreThanOneCommitteeBitSet, + /// The electra attestation has NO committee bit set + NoCommitteeBitSet, /// We have reached the maximum number of unique items that can be stored in a /// slot. This is a DoS protection function. ReachedMaxItemsPerSlot(usize), @@ -90,9 +201,6 @@ where /// Get a reference to the inner `HashMap`. fn get_map(&self) -> &HashMap; - /// Get a `Value` from `Self` based on `Key`, which is a hash of `Data`. - fn get_by_root(&self, root: &Self::Key) -> Option<&Self::Value>; - /// The number of items store in `Self`. fn len(&self) -> usize; @@ -112,13 +220,13 @@ where /// A collection of `Attestation` objects, keyed by their `attestation.data`. Enforces that all /// `attestation` are from the same slot. pub struct AggregatedAttestationMap { - map: HashMap>, + map: HashMap>, } impl AggregateMap for AggregatedAttestationMap { - type Key = AttestationDataRoot; + type Key = AttestationKeyRoot; type Value = Attestation; - type Data = AttestationData; + type Data = AttestationKey; /// Create an empty collection with the given `initial_capacity`. fn new(initial_capacity: usize) -> Self { @@ -133,45 +241,43 @@ impl AggregateMap for AggregatedAttestationMap { fn insert(&mut self, a: AttestationRef) -> Result { let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_CORE_INSERT); - let set_bits = match a { + let aggregation_bit = match a { AttestationRef::Base(att) => att .aggregation_bits .iter() .enumerate() - .filter(|(_i, bit)| *bit) - .map(|(i, _bit)| i) - .collect::>(), + .filter_map(|(i, bit)| if bit { Some(i) } else { None }) + .at_most_one() + .map_err(|iter| Error::MoreThanOneAggregationBitSet(iter.count()))? + .ok_or(Error::NoAggregationBitsSet)?, AttestationRef::Electra(att) => att .aggregation_bits .iter() .enumerate() - .filter(|(_i, bit)| *bit) - .map(|(i, _bit)| i) - .collect::>(), + .filter_map(|(i, bit)| if bit { Some(i) } else { None }) + .at_most_one() + .map_err(|iter| Error::MoreThanOneAggregationBitSet(iter.count()))? + .ok_or(Error::NoAggregationBitsSet)?, }; - let committee_index = set_bits - .first() - .copied() - .ok_or(Error::NoAggregationBitsSet)?; - - if set_bits.len() > 1 { - return Err(Error::MoreThanOneAggregationBitSet(set_bits.len())); - } - - let attestation_data_root = a.data().tree_hash_root(); + let attestation_key = AttestationKey::from_attestation_ref(a)?; + let attestation_data_root = attestation_key.tree_hash_root(); if let Some(existing_attestation) = self.map.get_mut(&attestation_data_root) { if existing_attestation - .get_aggregation_bit(committee_index) + .get_aggregation_bit(aggregation_bit) .map_err(|_| Error::InconsistentBitfieldLengths)? { - Ok(InsertOutcome::SignatureAlreadyKnown { committee_index }) + Ok(InsertOutcome::SignatureAlreadyKnown { + committee_index: aggregation_bit, + }) } else { let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_AGGREGATION); existing_attestation.aggregate(a); - Ok(InsertOutcome::SignatureAggregated { committee_index }) + Ok(InsertOutcome::SignatureAggregated { + committee_index: aggregation_bit, + }) } } else { if self.map.len() >= MAX_ATTESTATIONS_PER_SLOT { @@ -180,7 +286,9 @@ impl AggregateMap for AggregatedAttestationMap { self.map .insert(attestation_data_root, a.clone_as_attestation()); - Ok(InsertOutcome::NewItemInserted { committee_index }) + Ok(InsertOutcome::NewItemInserted { + committee_index: aggregation_bit, + }) } } @@ -195,11 +303,6 @@ impl AggregateMap for AggregatedAttestationMap { &self.map } - /// Returns an aggregated `Attestation` with the given `root`, if any. - fn get_by_root(&self, root: &Self::Key) -> Option<&Self::Value> { - self.map.get(root) - } - fn len(&self) -> usize { self.map.len() } @@ -306,11 +409,6 @@ impl AggregateMap for SyncContributionAggregateMap { &self.map } - /// Returns an aggregated `SyncCommitteeContribution` with the given `root`, if any. - fn get_by_root(&self, root: &SyncDataRoot) -> Option<&SyncCommitteeContribution> { - self.map.get(root) - } - fn len(&self) -> usize { self.map.len() } @@ -445,13 +543,6 @@ where .and_then(|map| map.get(data)) } - /// Returns an aggregated `T::Value` with the given `slot` and `root`, if any. - pub fn get_by_slot_and_root(&self, slot: Slot, root: &T::Key) -> Option { - self.maps - .get(&slot) - .and_then(|map| map.get_by_root(root).cloned()) - } - /// Iterate all items in all slots of `self`. pub fn iter(&self) -> impl Iterator { self.maps.values().flat_map(|map| map.get_map().values()) @@ -500,19 +591,30 @@ mod tests { use super::*; use ssz_types::BitList; use store::BitVector; + use tree_hash::TreeHash; use types::{ test_utils::{generate_deterministic_keypair, test_random_instance}, - Fork, Hash256, SyncCommitteeMessage, + Attestation, AttestationBase, AttestationElectra, Fork, Hash256, SyncCommitteeMessage, }; type E = types::MainnetEthSpec; - fn get_attestation(slot: Slot) -> Attestation { - let mut a: Attestation = test_random_instance(); - a.data_mut().slot = slot; - *a.aggregation_bits_base_mut().unwrap() = - BitList::with_capacity(4).expect("should create bitlist"); - a + fn get_attestation_base(slot: Slot) -> Attestation { + let mut a: AttestationBase = test_random_instance(); + a.data.slot = slot; + a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); + Attestation::Base(a) + } + + fn get_attestation_electra(slot: Slot) -> Attestation { + let mut a: AttestationElectra = test_random_instance(); + a.data.slot = slot; + a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); + a.committee_bits = BitVector::new(); + a.committee_bits + .set(0, true) + .expect("should set committee bit"); + Attestation::Electra(a) } fn get_sync_contribution(slot: Slot) -> SyncCommitteeContribution { @@ -555,10 +657,16 @@ mod tests { } fn unset_attestation_bit(a: &mut Attestation, i: usize) { - a.aggregation_bits_base_mut() - .unwrap() - .set(i, false) - .expect("should unset aggregation bit") + match a { + Attestation::Base(ref mut att) => att + .aggregation_bits + .set(i, false) + .expect("should unset aggregation bit"), + Attestation::Electra(ref mut att) => att + .aggregation_bits + .set(i, false) + .expect("should unset aggregation bit"), + } } fn unset_sync_contribution_bit(a: &mut SyncCommitteeContribution, i: usize) { @@ -579,8 +687,8 @@ mod tests { a.data().beacon_block_root == block_root } - fn key_from_attestation(a: &Attestation) -> AttestationData { - a.data().clone() + fn key_from_attestation(a: &Attestation) -> AttestationKey { + AttestationKey::from_attestation_ref(a.to_ref()).expect("should create attestation key") } fn mutate_sync_contribution_block_root( @@ -605,6 +713,45 @@ mod tests { SyncContributionData::from_contribution(a) } + #[test] + fn attestation_key_tree_hash_tests() { + let attestation_base = get_attestation_base(Slot::new(42)); + // for a base attestation, the tree_hash_root() of the key should be the same as the tree_hash_root() of the data + let attestation_key_base = AttestationKey::from_attestation_ref(attestation_base.to_ref()) + .expect("should create attestation key"); + assert_eq!( + attestation_key_base.tree_hash_root(), + attestation_base.data().tree_hash_root() + ); + let mut attestation_electra = get_attestation_electra(Slot::new(42)); + // for an electra attestation, the tree_hash_root() of the key should be different from the tree_hash_root() of the data + let attestation_key_electra = + AttestationKey::from_attestation_ref(attestation_electra.to_ref()) + .expect("should create attestation key"); + assert_ne!( + attestation_key_electra.tree_hash_root(), + attestation_electra.data().tree_hash_root() + ); + // for an electra attestation, the tree_hash_root() of the key should be dependent on which committee bit is set + let committe_bits = attestation_electra + .committee_bits_mut() + .expect("should get committee bits"); + committe_bits + .set(0, false) + .expect("should set committee bit"); + committe_bits + .set(1, true) + .expect("should set committee bit"); + let new_attestation_key_electra = + AttestationKey::from_attestation_ref(attestation_electra.to_ref()) + .expect("should create attestation key"); + // this new key should have a different tree_hash_root() than the previous key + assert_ne!( + attestation_key_electra.tree_hash_root(), + new_attestation_key_electra.tree_hash_root() + ); + } + macro_rules! test_suite { ( $mod_name: ident, @@ -800,8 +947,21 @@ mod tests { } test_suite! { - attestation_tests, - get_attestation, + attestation_tests_base, + get_attestation_base, + sign_attestation, + unset_attestation_bit, + mutate_attestation_block_root, + mutate_attestation_slot, + attestation_block_root_comparator, + key_from_attestation, + AggregatedAttestationMap, + MAX_ATTESTATIONS_PER_SLOT + } + + test_suite! { + attestation_tests_electra, + get_attestation_electra, sign_attestation, unset_attestation_bit, mutate_attestation_block_root, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index edfff4bf81e..bcf7582ebfd 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1374,7 +1374,7 @@ where // aggregate locally. let aggregate = self .chain - .get_aggregated_attestation(attestation.data()) + .get_aggregated_attestation_base(attestation.data()) .unwrap() .unwrap_or_else(|| { committee_attestations.iter().skip(1).fold( diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 8c9957db169..594872e2fff 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -1228,10 +1228,7 @@ async fn attesting_to_optimistic_head() { let get_aggregated_by_slot_and_root = || { rig.harness .chain - .get_aggregated_attestation_by_slot_and_root( - attestation.data().slot, - &attestation.data().tree_hash_root(), - ) + .get_aggregated_attestation_base(attestation.data()) }; /* diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 6cb8f6fe0b9..838f7233052 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3191,7 +3191,7 @@ pub fn serve( task_spawner.blocking_json_task(Priority::P0, move || { not_synced_filter?; chain - .get_aggregated_attestation_by_slot_and_root( + .get_pre_electra_aggregated_attestation_by_slot_and_root( query.slot, &query.attestation_data_root, ) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 595cc69f87c..9848840e96d 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -113,11 +113,15 @@ pub mod attesting_indices_electra { .map(|committee| (committee.index, committee)) .collect(); + let committee_count_per_slot = committees.len() as u64; + let mut participant_count = 0; for index in committee_indices { if let Some(&beacon_committee) = committees_map.get(&index) { - if aggregation_bits.len() != beacon_committee.committee.len() { - return Err(BeaconStateError::InvalidBitfield); + // This check is new to the spec's `process_attestation` in Electra. + if index >= committee_count_per_slot { + return Err(BeaconStateError::InvalidCommitteeIndex(index)); } + participant_count.safe_add_assign(beacon_committee.committee.len() as u64)?; let committee_attesters = beacon_committee .committee .iter() @@ -136,10 +140,13 @@ pub mod attesting_indices_electra { committee_offset.safe_add(beacon_committee.committee.len())?; } else { - return Err(Error::NoCommitteeFound); + return Err(Error::NoCommitteeFound(index)); } + } - // TODO(electra) what should we do when theres no committee found for a given index? + // This check is new to the spec's `process_attestation` in Electra. + if participant_count as usize != aggregation_bits.len() { + return Err(BeaconStateError::InvalidBitfield); } let mut indices = output.into_iter().collect_vec(); diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 0f7e8468488..8c8a81b90f2 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -67,9 +67,9 @@ pub struct Attestation { #[superstruct(only(Electra), partial_getter(rename = "aggregation_bits_electra"))] pub aggregation_bits: BitList, pub data: AttestationData, - pub signature: AggregateSignature, #[superstruct(only(Electra))] pub committee_bits: BitVector, + pub signature: AggregateSignature, } impl Decode for Attestation { @@ -92,6 +92,7 @@ impl Decode for Attestation { } } +// TODO(electra): think about how to handle fork variants here impl TestRandom for Attestation { fn random_for_test(rng: &mut impl RngCore) -> Self { let aggregation_bits: BitList = BitList::random_for_test(rng); diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 599c0bfc39c..d9c7a78537a 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -159,7 +159,8 @@ pub enum Error { IndexNotSupported(usize), InvalidFlagIndex(usize), MerkleTreeError(merkle_proof::MerkleTreeError), - NoCommitteeFound, + NoCommitteeFound(CommitteeIndex), + InvalidCommitteeIndex(CommitteeIndex), } /// Control whether an epoch-indexed field can be indexed at the next epoch or not. diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index cec4db2da51..14949e67531 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -408,6 +408,8 @@ impl EthSpec for MainnetEthSpec { pub struct MinimalEthSpec; impl EthSpec for MinimalEthSpec { + type MaxCommitteesPerSlot = U4; + type MaxValidatorsPerSlot = U8192; type SlotsPerEpoch = U8; type EpochsPerEth1VotingPeriod = U4; type SlotsPerHistoricalRoot = U64; @@ -432,8 +434,6 @@ impl EthSpec for MinimalEthSpec { SubnetBitfieldLength, SyncCommitteeSubnetCount, MaxValidatorsPerCommittee, - MaxCommitteesPerSlot, - MaxValidatorsPerSlot, GenesisEpoch, HistoricalRootsLimit, ValidatorRegistryLimit, diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 30db5c0e4a9..cbea78dabfc 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -111,11 +111,8 @@ type_name_generic!(LightClientUpdateDeneb, "LightClientUpdate"); type_name_generic!(PendingAttestation); type_name!(ProposerSlashing); type_name_generic!(SignedAggregateAndProof); -type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProofBase"); -type_name_generic!( - SignedAggregateAndProofElectra, - "SignedAggregateAndProofElectra" -); +type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProof"); +type_name_generic!(SignedAggregateAndProofElectra, "SignedAggregateAndProof"); type_name_generic!(SignedBeaconBlock); type_name!(SignedBeaconBlockHeader); type_name_generic!(SignedContributionAndProof); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 85d9362aaeb..fb8bdfcae71 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -219,7 +219,6 @@ mod ssz_static { use types::historical_summary::HistoricalSummary; use types::{AttesterSlashingBase, AttesterSlashingElectra, LightClientBootstrapAltair, *}; - ssz_static_test!(aggregate_and_proof, AggregateAndProof<_>); ssz_static_test!(attestation, Attestation<_>); ssz_static_test!(attestation_data, AttestationData); ssz_static_test!(beacon_block, SszStaticWithSpecHandler, BeaconBlock<_>); @@ -249,7 +248,7 @@ mod ssz_static { ssz_static_test!(voluntary_exit, VoluntaryExit); #[test] - fn signed_aggregate_and_proof() { + fn attester_slashing() { SszStaticHandler::, MinimalEthSpec>::pre_electra() .run(); SszStaticHandler::, MainnetEthSpec>::pre_electra() @@ -260,6 +259,36 @@ mod ssz_static { .run(); } + #[test] + fn signed_aggregate_and_proof() { + SszStaticHandler::, MinimalEthSpec>::pre_electra( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::pre_electra( + ) + .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); + } + + #[test] + fn aggregate_and_proof() { + SszStaticHandler::, MinimalEthSpec>::pre_electra() + .run(); + SszStaticHandler::, MainnetEthSpec>::pre_electra() + .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); + } + // BeaconBlockBody has no internal indicator of which fork it is for, so we test it separately. #[test] fn beacon_block_body() { @@ -283,22 +312,6 @@ mod ssz_static { .run(); } - #[test] - fn signed_aggregate_and_proof() { - SszStaticHandler::, MinimalEthSpec>::pre_electra( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::pre_electra( - ) - .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( - ) - .run(); - } - // Altair and later #[test] fn contribution_and_proof() { From ca0967119b168fa1fd05f35d3961e7447446cbf6 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Thu, 9 May 2024 17:10:04 -0500 Subject: [PATCH 041/227] Revert "Get `electra_op_pool` up to date (#5756)" (#5757) This reverts commit ab9e58aa3d0e6fe2175a4996a5de710e81152896. --- .../src/attestation_verification.rs | 5 +- beacon_node/beacon_chain/src/beacon_chain.rs | 38 +-- .../src/naive_aggregation_pool.rs | 282 ++++-------------- beacon_node/beacon_chain/src/test_utils.rs | 2 +- .../tests/payload_invalidation.rs | 5 +- beacon_node/http_api/src/lib.rs | 2 +- .../src/common/get_attesting_indices.rs | 15 +- consensus/types/src/attestation.rs | 3 +- consensus/types/src/beacon_state.rs | 3 +- consensus/types/src/eth_spec.rs | 4 +- testing/ef_tests/src/type_name.rs | 7 +- testing/ef_tests/tests/tests.rs | 49 ++- 12 files changed, 108 insertions(+), 307 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3d722a534be..62e65d5f87a 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -1309,11 +1309,10 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( attesting_indices_electra::get_indexed_attestation(&committees, att) .map(|attestation| (attestation, committees_per_slot)) .map_err(|e| { - let index = att.committee_index(); - if e == BlockOperationError::BeaconStateError(NoCommitteeFound(index)) { + if e == BlockOperationError::BeaconStateError(NoCommitteeFound) { Error::NoCommitteeForSlotAndIndex { slot: att.data.slot, - index, + index: att.committee_index(), } } else { Error::Invalid(e) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 331e04069fd..973dcaadb47 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1612,28 +1612,11 @@ impl BeaconChain { /// Returns an aggregated `Attestation`, if any, that has a matching `attestation.data`. /// /// The attestation will be obtained from `self.naive_aggregation_pool`. - pub fn get_aggregated_attestation_base( + pub fn get_aggregated_attestation( &self, data: &AttestationData, ) -> Result>, Error> { - let attestation_key = crate::naive_aggregation_pool::AttestationKey::new_base(data); - if let Some(attestation) = self.naive_aggregation_pool.read().get(&attestation_key) { - self.filter_optimistic_attestation(attestation) - .map(Option::Some) - } else { - Ok(None) - } - } - - // TODO(electra): call this function from the new beacon API method - pub fn get_aggregated_attestation_electra( - &self, - data: &AttestationData, - committee_index: CommitteeIndex, - ) -> Result>, Error> { - let attestation_key = - crate::naive_aggregation_pool::AttestationKey::new_electra(data, committee_index); - if let Some(attestation) = self.naive_aggregation_pool.read().get(&attestation_key) { + if let Some(attestation) = self.naive_aggregation_pool.read().get(data) { self.filter_optimistic_attestation(attestation) .map(Option::Some) } else { @@ -1645,21 +1628,16 @@ impl BeaconChain { /// `attestation.data.tree_hash_root()`. /// /// The attestation will be obtained from `self.naive_aggregation_pool`. - /// - /// NOTE: This function will *only* work with pre-electra attestations and it only - /// exists to support the pre-electra validator API method. - pub fn get_pre_electra_aggregated_attestation_by_slot_and_root( + pub fn get_aggregated_attestation_by_slot_and_root( &self, slot: Slot, attestation_data_root: &Hash256, ) -> Result>, Error> { - let attestation_key = - crate::naive_aggregation_pool::AttestationKey::new_base_from_slot_and_root( - slot, - *attestation_data_root, - ); - - if let Some(attestation) = self.naive_aggregation_pool.read().get(&attestation_key) { + if let Some(attestation) = self + .naive_aggregation_pool + .read() + .get_by_slot_and_root(slot, attestation_data_root) + { self.filter_optimistic_attestation(attestation) .map(Option::Some) } else { diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index 6c4f7cdae72..a12521cd171 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -1,9 +1,7 @@ use crate::metrics; use crate::observed_aggregates::AsReference; -use itertools::Itertools; -use smallvec::SmallVec; use std::collections::HashMap; -use tree_hash::{MerkleHasher, TreeHash, TreeHashType}; +use tree_hash::TreeHash; use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT; use types::slot_data::SlotData; use types::sync_committee_contribution::SyncContributionData; @@ -11,114 +9,9 @@ use types::{ Attestation, AttestationData, AttestationRef, EthSpec, Hash256, Slot, SyncCommitteeContribution, }; -type AttestationKeyRoot = Hash256; +type AttestationDataRoot = Hash256; type SyncDataRoot = Hash256; -/// Post-Electra, we need a new key for Attestations that includes the committee index -#[derive(Debug, Clone, PartialEq)] -pub struct AttestationKey { - data_root: Hash256, - committee_index: Option, - slot: Slot, -} - -// A custom implementation of `TreeHash` such that: -// AttestationKey(data, None).tree_hash_root() == data.tree_hash_root() -// AttestationKey(data, Some(index)).tree_hash_root() == (data, index).tree_hash_root() -// This is necessary because pre-Electra, the validator will ask for the tree_hash_root() -// of the `AttestationData` -impl TreeHash for AttestationKey { - fn tree_hash_type() -> TreeHashType { - TreeHashType::Container - } - - fn tree_hash_packed_encoding(&self) -> SmallVec<[u8; 32]> { - unreachable!("AttestationKey should never be packed.") - } - - fn tree_hash_packing_factor() -> usize { - unreachable!("AttestationKey should never be packed.") - } - - fn tree_hash_root(&self) -> Hash256 { - match self.committee_index { - None => self.data_root, // Return just the data root if no committee index is present - Some(index) => { - // Combine the hash of the data with the hash of the index - let mut hasher = MerkleHasher::with_leaves(2); - hasher - .write(self.data_root.as_bytes()) - .expect("should write data hash"); - hasher - .write(&index.to_le_bytes()) - .expect("should write index"); - hasher.finish().expect("should give tree hash") - } - } - } -} - -impl AttestationKey { - pub fn from_attestation_ref(attestation: AttestationRef) -> Result { - let slot = attestation.data().slot; - match attestation { - AttestationRef::Base(att) => Ok(Self { - data_root: att.data.tree_hash_root(), - committee_index: None, - slot, - }), - AttestationRef::Electra(att) => { - let committee_index = att - .committee_bits - .iter() - .enumerate() - .filter_map(|(i, bit)| if bit { Some(i) } else { None }) - .at_most_one() - .map_err(|_| Error::MoreThanOneCommitteeBitSet)? - .ok_or(Error::NoCommitteeBitSet)?; - - Ok(Self { - data_root: att.data.tree_hash_root(), - committee_index: Some(committee_index as u64), - slot, - }) - } - } - } - - pub fn new_base(data: &AttestationData) -> Self { - let slot = data.slot; - Self { - data_root: data.tree_hash_root(), - committee_index: None, - slot, - } - } - - pub fn new_electra(data: &AttestationData, committee_index: u64) -> Self { - let slot = data.slot; - Self { - data_root: data.tree_hash_root(), - committee_index: Some(committee_index), - slot, - } - } - - pub fn new_base_from_slot_and_root(slot: Slot, data_root: Hash256) -> Self { - Self { - data_root, - committee_index: None, - slot, - } - } -} - -impl SlotData for AttestationKey { - fn get_slot(&self) -> Slot { - self.slot - } -} - /// The number of slots that will be stored in the pool. /// /// For example, if `SLOTS_RETAINED == 3` and the pool is pruned at slot `6`, then all items @@ -156,10 +49,6 @@ pub enum Error { /// The given `aggregation_bits` field had more than one signature. The number of /// signatures found is included. MoreThanOneAggregationBitSet(usize), - /// The electra attestation has more than one committee bit set - MoreThanOneCommitteeBitSet, - /// The electra attestation has NO committee bit set - NoCommitteeBitSet, /// We have reached the maximum number of unique items that can be stored in a /// slot. This is a DoS protection function. ReachedMaxItemsPerSlot(usize), @@ -201,6 +90,9 @@ where /// Get a reference to the inner `HashMap`. fn get_map(&self) -> &HashMap; + /// Get a `Value` from `Self` based on `Key`, which is a hash of `Data`. + fn get_by_root(&self, root: &Self::Key) -> Option<&Self::Value>; + /// The number of items store in `Self`. fn len(&self) -> usize; @@ -220,13 +112,13 @@ where /// A collection of `Attestation` objects, keyed by their `attestation.data`. Enforces that all /// `attestation` are from the same slot. pub struct AggregatedAttestationMap { - map: HashMap>, + map: HashMap>, } impl AggregateMap for AggregatedAttestationMap { - type Key = AttestationKeyRoot; + type Key = AttestationDataRoot; type Value = Attestation; - type Data = AttestationKey; + type Data = AttestationData; /// Create an empty collection with the given `initial_capacity`. fn new(initial_capacity: usize) -> Self { @@ -241,43 +133,45 @@ impl AggregateMap for AggregatedAttestationMap { fn insert(&mut self, a: AttestationRef) -> Result { let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_CORE_INSERT); - let aggregation_bit = match a { + let set_bits = match a { AttestationRef::Base(att) => att .aggregation_bits .iter() .enumerate() - .filter_map(|(i, bit)| if bit { Some(i) } else { None }) - .at_most_one() - .map_err(|iter| Error::MoreThanOneAggregationBitSet(iter.count()))? - .ok_or(Error::NoAggregationBitsSet)?, + .filter(|(_i, bit)| *bit) + .map(|(i, _bit)| i) + .collect::>(), AttestationRef::Electra(att) => att .aggregation_bits .iter() .enumerate() - .filter_map(|(i, bit)| if bit { Some(i) } else { None }) - .at_most_one() - .map_err(|iter| Error::MoreThanOneAggregationBitSet(iter.count()))? - .ok_or(Error::NoAggregationBitsSet)?, + .filter(|(_i, bit)| *bit) + .map(|(i, _bit)| i) + .collect::>(), }; - let attestation_key = AttestationKey::from_attestation_ref(a)?; - let attestation_data_root = attestation_key.tree_hash_root(); + let committee_index = set_bits + .first() + .copied() + .ok_or(Error::NoAggregationBitsSet)?; + + if set_bits.len() > 1 { + return Err(Error::MoreThanOneAggregationBitSet(set_bits.len())); + } + + let attestation_data_root = a.data().tree_hash_root(); if let Some(existing_attestation) = self.map.get_mut(&attestation_data_root) { if existing_attestation - .get_aggregation_bit(aggregation_bit) + .get_aggregation_bit(committee_index) .map_err(|_| Error::InconsistentBitfieldLengths)? { - Ok(InsertOutcome::SignatureAlreadyKnown { - committee_index: aggregation_bit, - }) + Ok(InsertOutcome::SignatureAlreadyKnown { committee_index }) } else { let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_AGGREGATION); existing_attestation.aggregate(a); - Ok(InsertOutcome::SignatureAggregated { - committee_index: aggregation_bit, - }) + Ok(InsertOutcome::SignatureAggregated { committee_index }) } } else { if self.map.len() >= MAX_ATTESTATIONS_PER_SLOT { @@ -286,9 +180,7 @@ impl AggregateMap for AggregatedAttestationMap { self.map .insert(attestation_data_root, a.clone_as_attestation()); - Ok(InsertOutcome::NewItemInserted { - committee_index: aggregation_bit, - }) + Ok(InsertOutcome::NewItemInserted { committee_index }) } } @@ -303,6 +195,11 @@ impl AggregateMap for AggregatedAttestationMap { &self.map } + /// Returns an aggregated `Attestation` with the given `root`, if any. + fn get_by_root(&self, root: &Self::Key) -> Option<&Self::Value> { + self.map.get(root) + } + fn len(&self) -> usize { self.map.len() } @@ -409,6 +306,11 @@ impl AggregateMap for SyncContributionAggregateMap { &self.map } + /// Returns an aggregated `SyncCommitteeContribution` with the given `root`, if any. + fn get_by_root(&self, root: &SyncDataRoot) -> Option<&SyncCommitteeContribution> { + self.map.get(root) + } + fn len(&self) -> usize { self.map.len() } @@ -543,6 +445,13 @@ where .and_then(|map| map.get(data)) } + /// Returns an aggregated `T::Value` with the given `slot` and `root`, if any. + pub fn get_by_slot_and_root(&self, slot: Slot, root: &T::Key) -> Option { + self.maps + .get(&slot) + .and_then(|map| map.get_by_root(root).cloned()) + } + /// Iterate all items in all slots of `self`. pub fn iter(&self) -> impl Iterator { self.maps.values().flat_map(|map| map.get_map().values()) @@ -591,30 +500,19 @@ mod tests { use super::*; use ssz_types::BitList; use store::BitVector; - use tree_hash::TreeHash; use types::{ test_utils::{generate_deterministic_keypair, test_random_instance}, - Attestation, AttestationBase, AttestationElectra, Fork, Hash256, SyncCommitteeMessage, + Fork, Hash256, SyncCommitteeMessage, }; type E = types::MainnetEthSpec; - fn get_attestation_base(slot: Slot) -> Attestation { - let mut a: AttestationBase = test_random_instance(); - a.data.slot = slot; - a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); - Attestation::Base(a) - } - - fn get_attestation_electra(slot: Slot) -> Attestation { - let mut a: AttestationElectra = test_random_instance(); - a.data.slot = slot; - a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist"); - a.committee_bits = BitVector::new(); - a.committee_bits - .set(0, true) - .expect("should set committee bit"); - Attestation::Electra(a) + fn get_attestation(slot: Slot) -> Attestation { + let mut a: Attestation = test_random_instance(); + a.data_mut().slot = slot; + *a.aggregation_bits_base_mut().unwrap() = + BitList::with_capacity(4).expect("should create bitlist"); + a } fn get_sync_contribution(slot: Slot) -> SyncCommitteeContribution { @@ -657,16 +555,10 @@ mod tests { } fn unset_attestation_bit(a: &mut Attestation, i: usize) { - match a { - Attestation::Base(ref mut att) => att - .aggregation_bits - .set(i, false) - .expect("should unset aggregation bit"), - Attestation::Electra(ref mut att) => att - .aggregation_bits - .set(i, false) - .expect("should unset aggregation bit"), - } + a.aggregation_bits_base_mut() + .unwrap() + .set(i, false) + .expect("should unset aggregation bit") } fn unset_sync_contribution_bit(a: &mut SyncCommitteeContribution, i: usize) { @@ -687,8 +579,8 @@ mod tests { a.data().beacon_block_root == block_root } - fn key_from_attestation(a: &Attestation) -> AttestationKey { - AttestationKey::from_attestation_ref(a.to_ref()).expect("should create attestation key") + fn key_from_attestation(a: &Attestation) -> AttestationData { + a.data().clone() } fn mutate_sync_contribution_block_root( @@ -713,45 +605,6 @@ mod tests { SyncContributionData::from_contribution(a) } - #[test] - fn attestation_key_tree_hash_tests() { - let attestation_base = get_attestation_base(Slot::new(42)); - // for a base attestation, the tree_hash_root() of the key should be the same as the tree_hash_root() of the data - let attestation_key_base = AttestationKey::from_attestation_ref(attestation_base.to_ref()) - .expect("should create attestation key"); - assert_eq!( - attestation_key_base.tree_hash_root(), - attestation_base.data().tree_hash_root() - ); - let mut attestation_electra = get_attestation_electra(Slot::new(42)); - // for an electra attestation, the tree_hash_root() of the key should be different from the tree_hash_root() of the data - let attestation_key_electra = - AttestationKey::from_attestation_ref(attestation_electra.to_ref()) - .expect("should create attestation key"); - assert_ne!( - attestation_key_electra.tree_hash_root(), - attestation_electra.data().tree_hash_root() - ); - // for an electra attestation, the tree_hash_root() of the key should be dependent on which committee bit is set - let committe_bits = attestation_electra - .committee_bits_mut() - .expect("should get committee bits"); - committe_bits - .set(0, false) - .expect("should set committee bit"); - committe_bits - .set(1, true) - .expect("should set committee bit"); - let new_attestation_key_electra = - AttestationKey::from_attestation_ref(attestation_electra.to_ref()) - .expect("should create attestation key"); - // this new key should have a different tree_hash_root() than the previous key - assert_ne!( - attestation_key_electra.tree_hash_root(), - new_attestation_key_electra.tree_hash_root() - ); - } - macro_rules! test_suite { ( $mod_name: ident, @@ -947,21 +800,8 @@ mod tests { } test_suite! { - attestation_tests_base, - get_attestation_base, - sign_attestation, - unset_attestation_bit, - mutate_attestation_block_root, - mutate_attestation_slot, - attestation_block_root_comparator, - key_from_attestation, - AggregatedAttestationMap, - MAX_ATTESTATIONS_PER_SLOT - } - - test_suite! { - attestation_tests_electra, - get_attestation_electra, + attestation_tests, + get_attestation, sign_attestation, unset_attestation_bit, mutate_attestation_block_root, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index bcf7582ebfd..edfff4bf81e 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1374,7 +1374,7 @@ where // aggregate locally. let aggregate = self .chain - .get_aggregated_attestation_base(attestation.data()) + .get_aggregated_attestation(attestation.data()) .unwrap() .unwrap_or_else(|| { committee_attestations.iter().skip(1).fold( diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 594872e2fff..8c9957db169 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -1228,7 +1228,10 @@ async fn attesting_to_optimistic_head() { let get_aggregated_by_slot_and_root = || { rig.harness .chain - .get_aggregated_attestation_base(attestation.data()) + .get_aggregated_attestation_by_slot_and_root( + attestation.data().slot, + &attestation.data().tree_hash_root(), + ) }; /* diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 838f7233052..6cb8f6fe0b9 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3191,7 +3191,7 @@ pub fn serve( task_spawner.blocking_json_task(Priority::P0, move || { not_synced_filter?; chain - .get_pre_electra_aggregated_attestation_by_slot_and_root( + .get_aggregated_attestation_by_slot_and_root( query.slot, &query.attestation_data_root, ) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 9848840e96d..595cc69f87c 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -113,15 +113,11 @@ pub mod attesting_indices_electra { .map(|committee| (committee.index, committee)) .collect(); - let committee_count_per_slot = committees.len() as u64; - let mut participant_count = 0; for index in committee_indices { if let Some(&beacon_committee) = committees_map.get(&index) { - // This check is new to the spec's `process_attestation` in Electra. - if index >= committee_count_per_slot { - return Err(BeaconStateError::InvalidCommitteeIndex(index)); + if aggregation_bits.len() != beacon_committee.committee.len() { + return Err(BeaconStateError::InvalidBitfield); } - participant_count.safe_add_assign(beacon_committee.committee.len() as u64)?; let committee_attesters = beacon_committee .committee .iter() @@ -140,13 +136,10 @@ pub mod attesting_indices_electra { committee_offset.safe_add(beacon_committee.committee.len())?; } else { - return Err(Error::NoCommitteeFound(index)); + return Err(Error::NoCommitteeFound); } - } - // This check is new to the spec's `process_attestation` in Electra. - if participant_count as usize != aggregation_bits.len() { - return Err(BeaconStateError::InvalidBitfield); + // TODO(electra) what should we do when theres no committee found for a given index? } let mut indices = output.into_iter().collect_vec(); diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 8c8a81b90f2..0f7e8468488 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -67,9 +67,9 @@ pub struct Attestation { #[superstruct(only(Electra), partial_getter(rename = "aggregation_bits_electra"))] pub aggregation_bits: BitList, pub data: AttestationData, + pub signature: AggregateSignature, #[superstruct(only(Electra))] pub committee_bits: BitVector, - pub signature: AggregateSignature, } impl Decode for Attestation { @@ -92,7 +92,6 @@ impl Decode for Attestation { } } -// TODO(electra): think about how to handle fork variants here impl TestRandom for Attestation { fn random_for_test(rng: &mut impl RngCore) -> Self { let aggregation_bits: BitList = BitList::random_for_test(rng); diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index d9c7a78537a..599c0bfc39c 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -159,8 +159,7 @@ pub enum Error { IndexNotSupported(usize), InvalidFlagIndex(usize), MerkleTreeError(merkle_proof::MerkleTreeError), - NoCommitteeFound(CommitteeIndex), - InvalidCommitteeIndex(CommitteeIndex), + NoCommitteeFound, } /// Control whether an epoch-indexed field can be indexed at the next epoch or not. diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 14949e67531..cec4db2da51 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -408,8 +408,6 @@ impl EthSpec for MainnetEthSpec { pub struct MinimalEthSpec; impl EthSpec for MinimalEthSpec { - type MaxCommitteesPerSlot = U4; - type MaxValidatorsPerSlot = U8192; type SlotsPerEpoch = U8; type EpochsPerEth1VotingPeriod = U4; type SlotsPerHistoricalRoot = U64; @@ -434,6 +432,8 @@ impl EthSpec for MinimalEthSpec { SubnetBitfieldLength, SyncCommitteeSubnetCount, MaxValidatorsPerCommittee, + MaxCommitteesPerSlot, + MaxValidatorsPerSlot, GenesisEpoch, HistoricalRootsLimit, ValidatorRegistryLimit, diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index cbea78dabfc..30db5c0e4a9 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -111,8 +111,11 @@ type_name_generic!(LightClientUpdateDeneb, "LightClientUpdate"); type_name_generic!(PendingAttestation); type_name!(ProposerSlashing); type_name_generic!(SignedAggregateAndProof); -type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProof"); -type_name_generic!(SignedAggregateAndProofElectra, "SignedAggregateAndProof"); +type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProofBase"); +type_name_generic!( + SignedAggregateAndProofElectra, + "SignedAggregateAndProofElectra" +); type_name_generic!(SignedBeaconBlock); type_name!(SignedBeaconBlockHeader); type_name_generic!(SignedContributionAndProof); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index fb8bdfcae71..85d9362aaeb 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -219,6 +219,7 @@ mod ssz_static { use types::historical_summary::HistoricalSummary; use types::{AttesterSlashingBase, AttesterSlashingElectra, LightClientBootstrapAltair, *}; + ssz_static_test!(aggregate_and_proof, AggregateAndProof<_>); ssz_static_test!(attestation, Attestation<_>); ssz_static_test!(attestation_data, AttestationData); ssz_static_test!(beacon_block, SszStaticWithSpecHandler, BeaconBlock<_>); @@ -248,7 +249,7 @@ mod ssz_static { ssz_static_test!(voluntary_exit, VoluntaryExit); #[test] - fn attester_slashing() { + fn signed_aggregate_and_proof() { SszStaticHandler::, MinimalEthSpec>::pre_electra() .run(); SszStaticHandler::, MainnetEthSpec>::pre_electra() @@ -259,36 +260,6 @@ mod ssz_static { .run(); } - #[test] - fn signed_aggregate_and_proof() { - SszStaticHandler::, MinimalEthSpec>::pre_electra( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::pre_electra( - ) - .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( - ) - .run(); - } - - #[test] - fn aggregate_and_proof() { - SszStaticHandler::, MinimalEthSpec>::pre_electra() - .run(); - SszStaticHandler::, MainnetEthSpec>::pre_electra() - .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( - ) - .run(); - } - // BeaconBlockBody has no internal indicator of which fork it is for, so we test it separately. #[test] fn beacon_block_body() { @@ -312,6 +283,22 @@ mod ssz_static { .run(); } + #[test] + fn signed_aggregate_and_proof() { + SszStaticHandler::, MinimalEthSpec>::pre_electra( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::pre_electra( + ) + .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); + } + // Altair and later #[test] fn contribution_and_proof() { From eb2260222c45cdd260f32d9ea650ae0a9ec46de0 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 9 May 2024 18:59:53 -0400 Subject: [PATCH 042/227] update tests --- .../src/common/initiate_validator_exit.rs | 2 +- consensus/types/src/chain_spec.rs | 8 ++ testing/ef_tests/Makefile | 2 +- testing/ef_tests/src/cases/operations.rs | 12 +- testing/ef_tests/src/handler.rs | 35 ++--- testing/ef_tests/tests/tests.rs | 124 ++++++++++++++---- 6 files changed, 127 insertions(+), 56 deletions(-) diff --git a/consensus/state_processing/src/common/initiate_validator_exit.rs b/consensus/state_processing/src/common/initiate_validator_exit.rs index 8874e9ed4b8..f917f66d428 100644 --- a/consensus/state_processing/src/common/initiate_validator_exit.rs +++ b/consensus/state_processing/src/common/initiate_validator_exit.rs @@ -20,7 +20,7 @@ pub fn initiate_validator_exit( // Compute exit queue epoch let exit_queue_epoch = if state.fork_name_unchecked() >= ForkName::Electra { - let effective_balance = state.get_validator(index)?.effective_balance; + let effective_balance = state.get_effective_balance(index)?; state.compute_exit_epoch_and_update_churn(effective_balance, spec)? } else { let delayed_epoch = state.compute_activation_exit_epoch(state.current_epoch(), spec)?; diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 8ffafe6266d..1cda156d8c2 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -865,6 +865,14 @@ impl ChainSpec { electra_fork_epoch: None, max_pending_partials_per_withdrawals_sweep: u64::checked_pow(2, 0) .expect("pow does not overflow"), + min_per_epoch_churn_limit_electra: option_wrapper(|| { + u64::checked_pow(2, 6)?.checked_mul(u64::checked_pow(10, 9)?) + }) + .expect("calculation does not overflow"), + max_per_epoch_activation_exit_churn_limit: option_wrapper(|| { + u64::checked_pow(2, 7)?.checked_mul(u64::checked_pow(10, 9)?) + }) + .expect("calculation does not overflow"), // Other network_id: 2, // lighthouse testnet network id deposit_chain_id: 5, diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index 508c284275a..5dc3d2a0404 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,4 +1,4 @@ -TESTS_TAG := v1.4.0-beta.6 +TESTS_TAG := v1.5.0-alpha.2 TESTS = general minimal mainnet TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index b922668c0a0..f7adcbd75ee 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -290,7 +290,7 @@ impl Operation for BeaconBlockBody> { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name != ForkName::Base && fork_name != ForkName::Altair + fork_name >= ForkName::Bellatrix } fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -331,7 +331,7 @@ impl Operation for BeaconBlockBody> { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name != ForkName::Base && fork_name != ForkName::Altair + fork_name >= ForkName::Bellatrix } fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -383,9 +383,7 @@ impl Operation for WithdrawalsPayload { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name != ForkName::Base - && fork_name != ForkName::Altair - && fork_name != ForkName::Bellatrix + fork_name >= ForkName::Capella } fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -417,9 +415,7 @@ impl Operation for SignedBlsToExecutionChange { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name != ForkName::Base - && fork_name != ForkName::Altair - && fork_name != ForkName::Bellatrix + fork_name >= ForkName::Capella } fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index c1e7386bc45..2f861af71e4 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -20,9 +20,8 @@ pub trait Handler { // Add forks here to exclude them from EF spec testing. Helpful for adding future or // unspecified forks. - // TODO(electra): Enable Electra once spec tests are available. fn disabled_forks(&self) -> Vec { - vec![ForkName::Electra] + vec![] } fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool { @@ -245,6 +244,14 @@ impl SszStaticHandler { Self::for_forks(ForkName::list_all()[3..].to_vec()) } + pub fn deneb_and_later() -> Self { + Self::for_forks(ForkName::list_all()[4..].to_vec()) + } + + pub fn electra_and_later() -> Self { + Self::for_forks(ForkName::list_all()[5..].to_vec()) + } + pub fn pre_electra() -> Self { Self::for_forks(ForkName::list_all()[0..5].to_vec()) } @@ -577,7 +584,7 @@ impl Handler for ForkChoiceHandler { // No FCU override tests prior to bellatrix. if self.handler_name == "should_override_forkchoice_update" - && (fork_name == ForkName::Base || fork_name == ForkName::Altair) + && fork_name >= ForkName::Bellatrix { return false; } @@ -613,9 +620,7 @@ impl Handler for OptimisticSyncHandler { } fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool { - fork_name != ForkName::Base - && fork_name != ForkName::Altair - && cfg!(not(feature = "fake_crypto")) + fork_name >= ForkName::Bellatrix && cfg!(not(feature = "fake_crypto")) } } @@ -798,13 +803,12 @@ impl Handler for MerkleProofValidityHandler { "single_merkle_proof".into() } - fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool { - fork_name != ForkName::Base - // Test is skipped due to some changes in the Capella light client - // spec. - // - // https://github.com/sigp/lighthouse/issues/4022 - && fork_name != ForkName::Capella && fork_name != ForkName::Deneb + fn is_enabled_for_fork(&self, _fork_name: ForkName) -> bool { + // Test is skipped due to some changes in the Capella light client + // spec. + // + // https://github.com/sigp/lighthouse/issues/4022 + false } } @@ -829,10 +833,7 @@ impl Handler for KzgInclusionMerkleProofValidityHandler bool { // Enabled in Deneb - fork_name != ForkName::Base - && fork_name != ForkName::Altair - && fork_name != ForkName::Bellatrix - && fork_name != ForkName::Capella + fork_name >= ForkName::Deneb } } diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index fb8bdfcae71..1947dd4b419 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -14,6 +14,10 @@ fn check_typenum_values() { E::SlotsPerEth1VotingPeriod::to_u64(), E::EpochsPerEth1VotingPeriod::to_u64() * E::SlotsPerEpoch::to_u64() ); + assert_eq!( + E::MaxValidatorsPerSlot::to_u64(), + E::MaxCommitteesPerSlot::to_u64() * E::MaxValidatorsPerCommittee::to_u64() + ); } #[test] @@ -217,7 +221,11 @@ mod ssz_static { use ef_tests::{Handler, SszStaticHandler, SszStaticTHCHandler, SszStaticWithSpecHandler}; use types::blob_sidecar::BlobIdentifier; use types::historical_summary::HistoricalSummary; - use types::{AttesterSlashingBase, AttesterSlashingElectra, LightClientBootstrapAltair, *}; + use types::{ + AttesterSlashingBase, AttesterSlashingElectra, Consolidation, DepositReceipt, + ExecutionLayerWithdrawalRequest, LightClientBootstrapAltair, PendingBalanceDeposit, + PendingPartialWithdrawal, *, + }; ssz_static_test!(attestation, Attestation<_>); ssz_static_test!(attestation_data, AttestationData); @@ -310,6 +318,10 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only() + .run(); } // Altair and later @@ -344,6 +356,10 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only() + .run(); } // LightClientHeader has no internal indicator of which fork it is for, so we test it separately. @@ -369,35 +385,27 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); } // LightClientOptimisticUpdate has no internal indicator of which fork it is for, so we test it separately. #[test] fn light_client_optimistic_update() { - SszStaticHandler::, MinimalEthSpec>::altair_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::altair_only( - ) - .run(); - SszStaticHandler::, MinimalEthSpec>::bellatrix_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::bellatrix_only( - ) - .run(); - SszStaticHandler::, MinimalEthSpec>::capella_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::capella_only( - ) - .run(); - SszStaticHandler::, MinimalEthSpec>::deneb_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::deneb_only( - ) - .run(); + SszStaticHandler::, MinimalEthSpec>::altair_only().run(); + SszStaticHandler::, MainnetEthSpec>::altair_only().run(); + SszStaticHandler::, MinimalEthSpec>::bellatrix_only().run(); + SszStaticHandler::, MainnetEthSpec>::bellatrix_only().run(); + SszStaticHandler::, MinimalEthSpec>::capella_only().run(); + SszStaticHandler::, MainnetEthSpec>::capella_only().run(); + SszStaticHandler::, MinimalEthSpec>::deneb_only().run(); + SszStaticHandler::, MainnetEthSpec>::deneb_only().run(); + SszStaticHandler::, MinimalEthSpec>::electra_only().run(); + SszStaticHandler::, MainnetEthSpec>::electra_only().run(); } // LightClientFinalityUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -427,6 +435,12 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::deneb_only( ) .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); } // LightClientUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -450,6 +464,12 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); } #[test] @@ -503,6 +523,10 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only() + .run(); } #[test] @@ -519,6 +543,10 @@ mod ssz_static { ::deneb_only().run(); SszStaticHandler::, MainnetEthSpec> ::deneb_only().run(); + SszStaticHandler::, MinimalEthSpec> + ::electra_only().run(); + SszStaticHandler::, MainnetEthSpec> + ::electra_only().run(); } #[test] @@ -541,14 +569,14 @@ mod ssz_static { #[test] fn blob_sidecar() { - SszStaticHandler::, MinimalEthSpec>::deneb_only().run(); - SszStaticHandler::, MainnetEthSpec>::deneb_only().run(); + SszStaticHandler::, MinimalEthSpec>::deneb_and_later().run(); + SszStaticHandler::, MainnetEthSpec>::deneb_and_later().run(); } #[test] fn blob_identifier() { - SszStaticHandler::::deneb_only().run(); - SszStaticHandler::::deneb_only().run(); + SszStaticHandler::::deneb_and_later().run(); + SszStaticHandler::::deneb_and_later().run(); } #[test] @@ -556,6 +584,44 @@ mod ssz_static { SszStaticHandler::::capella_and_later().run(); SszStaticHandler::::capella_and_later().run(); } + + #[test] + fn consolidation() { + SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); + } + + #[test] + fn deposit_receipt() { + SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); + } + + #[test] + fn execution_layer_withdrawal_request() { + SszStaticHandler::::electra_and_later() + .run(); + SszStaticHandler::::electra_and_later() + .run(); + } + + #[test] + fn pending_balance_deposit() { + SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); + } + + #[test] + fn pending_consolidation() { + SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); + } + + #[test] + fn pending_partial_withdrawal() { + SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); + } } #[test] From 411fcee2ac879533870c4116b28e569a0012020c Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Fri, 10 May 2024 02:56:20 +0200 Subject: [PATCH 043/227] Compute on chain aggregate impl (#5752) * add compute_on_chain_agg impl to op pool changes * fmt * get op pool tests to pass --- .../operation_pool/src/attestation_storage.rs | 79 +++++++++++++++++-- beacon_node/operation_pool/src/lib.rs | 59 ++++++++++++-- 2 files changed, 124 insertions(+), 14 deletions(-) diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index 5cf93e642c9..6e04af01b9c 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -187,6 +187,13 @@ impl CompactIndexedAttestation { _ => (), } } + + pub fn committee_index(&self) -> u64 { + match self { + CompactIndexedAttestation::Base(att) => att.index, + CompactIndexedAttestation::Electra(att) => att.committee_index(), + } + } } impl CompactIndexedAttestationBase { @@ -276,25 +283,34 @@ impl AttestationMap { let Some(attestation_map) = self.checkpoint_map.get_mut(&checkpoint_key) else { return; }; - for (compact_attestation_data, compact_indexed_attestations) in - attestation_map.attestations.iter_mut() - { + for (_, compact_indexed_attestations) in attestation_map.attestations.iter_mut() { let unaggregated_attestations = std::mem::take(compact_indexed_attestations); - let mut aggregated_attestations = vec![]; + let mut aggregated_attestations: Vec> = vec![]; // Aggregate the best attestations for each committee and leave the rest. - let mut best_attestations_by_committee = BTreeMap::new(); + let mut best_attestations_by_committee: BTreeMap> = + BTreeMap::new(); for committee_attestation in unaggregated_attestations { // TODO(electra) // compare to best attestations by committee // could probably use `.entry` here if let Some(existing_attestation) = - best_attestations_by_committee.get_mut(committee_attestation.committee_index()) + best_attestations_by_committee.get_mut(&committee_attestation.committee_index()) { // compare and swap, put the discarded one straight into // `aggregated_attestations` in case we have room to pack it without // cross-committee aggregation + if existing_attestation.should_aggregate(&committee_attestation) { + existing_attestation.aggregate(&committee_attestation); + + best_attestations_by_committee.insert( + committee_attestation.committee_index(), + committee_attestation, + ); + } else { + aggregated_attestations.push(committee_attestation); + } } else { best_attestations_by_committee.insert( committee_attestation.committee_index(), @@ -305,11 +321,62 @@ impl AttestationMap { // TODO(electra): aggregate all the best attestations by committee // (use btreemap sort order to get order by committee index) + aggregated_attestations.extend(Self::compute_on_chain_aggregate( + best_attestations_by_committee, + )); *compact_indexed_attestations = aggregated_attestations; } } + // TODO(electra) unwraps in this function should be cleaned up + // also in general this could be a bit more elegant + pub fn compute_on_chain_aggregate( + mut attestations_by_committee: BTreeMap>, + ) -> Vec> { + let mut aggregated_attestations = vec![]; + if let Some((_, on_chain_aggregate)) = attestations_by_committee.pop_first() { + match on_chain_aggregate { + CompactIndexedAttestation::Base(a) => { + aggregated_attestations.push(CompactIndexedAttestation::Base(a)); + aggregated_attestations.extend( + attestations_by_committee + .values() + .map(|a| { + CompactIndexedAttestation::Base(CompactIndexedAttestationBase { + attesting_indices: a.attesting_indices().clone(), + aggregation_bits: a.aggregation_bits_base().unwrap().clone(), + signature: a.signature().clone(), + index: *a.index(), + }) + }) + .collect::>>(), + ); + } + CompactIndexedAttestation::Electra(mut a) => { + for (_, attestation) in attestations_by_committee.iter_mut() { + let new_committee_bits = a + .committee_bits + .union(attestation.committee_bits().unwrap()); + a.aggregate(attestation.as_electra().unwrap()); + + a = CompactIndexedAttestationElectra { + attesting_indices: a.attesting_indices.clone(), + aggregation_bits: a.aggregation_bits.clone(), + signature: a.signature.clone(), + index: a.index, + committee_bits: new_committee_bits, + }; + } + + aggregated_attestations.push(CompactIndexedAttestation::Electra(a)); + } + } + } + + aggregated_attestations + } + /// Iterate all attestations matching the given `checkpoint_key`. pub fn get_attestations<'a>( &'a self, diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 1d83ecd4651..4f247e6bf20 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -1246,7 +1246,17 @@ mod release_tests { let num_big = target_committee_size / big_step_size; let stats = op_pool.attestation_stats(); - assert_eq!(stats.num_attestation_data, committees.len()); + let fork_name = state.fork_name_unchecked(); + + match fork_name { + ForkName::Electra => { + assert_eq!(stats.num_attestation_data, 1); + } + _ => { + assert_eq!(stats.num_attestation_data, committees.len()); + } + }; + assert_eq!( stats.num_attestations, (num_small + num_big) * committees.len() @@ -1257,11 +1267,27 @@ mod release_tests { let best_attestations = op_pool .get_attestations(&state, |_| true, |_| true, spec) .expect("should have best attestations"); - assert_eq!(best_attestations.len(), max_attestations); + match fork_name { + ForkName::Electra => { + assert_eq!(best_attestations.len(), 8); + } + _ => { + assert_eq!(best_attestations.len(), max_attestations); + } + }; // All the best attestations should be signed by at least `big_step_size` (4) validators. for att in &best_attestations { - assert!(att.num_set_aggregation_bits() >= big_step_size); + match fork_name { + ForkName::Electra => { + // TODO(electra) some attestations only have 2 or 3 agg bits set + // others have 5 + assert!(att.num_set_aggregation_bits() >= 2); + } + _ => { + assert!(att.num_set_aggregation_bits() >= big_step_size); + } + }; } } @@ -1340,11 +1366,20 @@ mod release_tests { let num_small = target_committee_size / small_step_size; let num_big = target_committee_size / big_step_size; + let fork_name = state.fork_name_unchecked(); + + match fork_name { + ForkName::Electra => { + assert_eq!(op_pool.attestation_stats().num_attestation_data, 1); + } + _ => { + assert_eq!( + op_pool.attestation_stats().num_attestation_data, + committees.len() + ); + } + }; - assert_eq!( - op_pool.attestation_stats().num_attestation_data, - committees.len() - ); assert_eq!( op_pool.num_attestations(), (num_small + num_big) * committees.len() @@ -1355,7 +1390,15 @@ mod release_tests { let best_attestations = op_pool .get_attestations(&state, |_| true, |_| true, spec) .expect("should have valid best attestations"); - assert_eq!(best_attestations.len(), max_attestations); + + match fork_name { + ForkName::Electra => { + assert_eq!(best_attestations.len(), 8); + } + _ => { + assert_eq!(best_attestations.len(), max_attestations); + } + }; let total_active_balance = state.get_total_active_balance().unwrap(); From 6477eecc65812990d3ea09f6b6782504546e0784 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 9 May 2024 21:19:59 -0400 Subject: [PATCH 044/227] update beacon api aggregate attestationendpoint --- beacon_node/http_api/src/lib.rs | 46 +++++++++++++++++---------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index f8b6fcfa3c1..82a6c9847b0 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3191,36 +3191,38 @@ pub fn serve( task_spawner: TaskSpawner, chain: Arc>| { task_spawner.blocking_json_task(Priority::P0, move || { - if endpoint_version == V2 { - if query.committee_index.is_none() { + not_synced_filter?; + let res = if endpoint_version == V2 { + let Some(committee_index) = query.committee_index else { return Err(warp_utils::reject::custom_bad_request( "missing committee index".to_string(), )); - } + }; + chain.get_aggregated_attestation_electra( + &query.attestation_data_root, + committee_index, + ) } else if endpoint_version == V1 { // Do nothing - } else { - return Err(unsupported_version_rejection(endpoint_version)); - } - //TODO(electra) pass the index into the next method. - not_synced_filter?; - chain - .get_pre_electra_aggregated_attestation_by_slot_and_root( + chain.get_pre_electra_aggregated_attestation_by_slot_and_root( query.slot, &query.attestation_data_root, ) - .map_err(|e| { - warp_utils::reject::custom_bad_request(format!( - "unable to fetch aggregate: {:?}", - e - )) - })? - .map(api_types::GenericResponse::from) - .ok_or_else(|| { - warp_utils::reject::custom_not_found( - "no matching aggregate found".to_string(), - ) - }) + } else { + return Err(unsupported_version_rejection(endpoint_version)); + }; + res.map_err(|e| { + warp_utils::reject::custom_bad_request(format!( + "unable to fetch aggregate: {:?}", + e + )) + })? + .map(api_types::GenericResponse::from) + .ok_or_else(|| { + warp_utils::reject::custom_not_found( + "no matching aggregate found".to_string(), + ) + }) }) }, ); From e4485570f22a340b7111f9a688705f2cb02b81e5 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 9 May 2024 21:29:31 -0400 Subject: [PATCH 045/227] update the naive agg pool interface (#5760) --- beacon_node/beacon_chain/src/beacon_chain.rs | 11 +++++++---- .../beacon_chain/src/naive_aggregation_pool.rs | 10 +++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 331e04069fd..9cbb46afe93 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1625,14 +1625,17 @@ impl BeaconChain { } } - // TODO(electra): call this function from the new beacon API method pub fn get_aggregated_attestation_electra( &self, - data: &AttestationData, + slot: Slot, + attestation_data_root: &Hash256, committee_index: CommitteeIndex, ) -> Result>, Error> { - let attestation_key = - crate::naive_aggregation_pool::AttestationKey::new_electra(data, committee_index); + let attestation_key = crate::naive_aggregation_pool::AttestationKey::new_electra( + slot, + *attestation_data_root, + committee_index, + ); if let Some(attestation) = self.naive_aggregation_pool.read().get(&attestation_key) { self.filter_optimistic_attestation(attestation) .map(Option::Some) diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index 6c4f7cdae72..a1c736cd0ef 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -8,7 +8,8 @@ use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT; use types::slot_data::SlotData; use types::sync_committee_contribution::SyncContributionData; use types::{ - Attestation, AttestationData, AttestationRef, EthSpec, Hash256, Slot, SyncCommitteeContribution, + Attestation, AttestationData, AttestationRef, CommitteeIndex, EthSpec, Hash256, Slot, + SyncCommitteeContribution, }; type AttestationKeyRoot = Hash256; @@ -18,7 +19,7 @@ type SyncDataRoot = Hash256; #[derive(Debug, Clone, PartialEq)] pub struct AttestationKey { data_root: Hash256, - committee_index: Option, + committee_index: Option, slot: Slot, } @@ -95,10 +96,9 @@ impl AttestationKey { } } - pub fn new_electra(data: &AttestationData, committee_index: u64) -> Self { - let slot = data.slot; + pub fn new_electra(slot: Slot, data_root: Hash256, committee_index: CommitteeIndex) -> Self { Self { - data_root: data.tree_hash_root(), + data_root, committee_index: Some(committee_index), slot, } From d505c04507bbd6a862e5ae11ad9e92530d6a03c0 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 9 May 2024 21:33:48 -0400 Subject: [PATCH 046/227] updates after merge --- beacon_node/execution_layer/src/engine_api.rs | 2 +- beacon_node/http_api/src/lib.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 09e36be41f4..6bfd9997f43 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -644,7 +644,7 @@ impl ExecutionPayloadBodyV1 { "block {} is post-electra but payload body doesn't have withdrawals/deposit_receipts/withdrawal_requests \ Check that ELs are returning receipts and withdrawal_requests in getPayloadBody requests", header.block_hash - )) + )); }; Ok(ExecutionPayload::Electra(ExecutionPayloadElectra { parent_hash: header.parent_hash, diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 82a6c9847b0..5d9a50ed07a 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3199,6 +3199,7 @@ pub fn serve( )); }; chain.get_aggregated_attestation_electra( + query.slot, &query.attestation_data_root, committee_index, ) From 437e8516cd949481f374fbbc83e7ea6bf8236f62 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 10 May 2024 12:29:57 +1000 Subject: [PATCH 047/227] Fix bugs in cross-committee aggregation --- .../operation_pool/src/attestation_storage.rs | 193 ++++++++++-------- beacon_node/operation_pool/src/lib.rs | 6 +- 2 files changed, 111 insertions(+), 88 deletions(-) diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index 6e04af01b9c..4cc783e64f2 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, HashMap}; use types::{ attestation::{AttestationBase, AttestationElectra}, superstruct, AggregateSignature, Attestation, AttestationData, BeaconState, BitList, BitVector, - Checkpoint, Epoch, EthSpec, Hash256, Slot, + Checkpoint, Epoch, EthSpec, Hash256, Slot, Unsigned, }; #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] @@ -30,7 +30,6 @@ pub struct CompactIndexedAttestation { #[superstruct(only(Electra), partial_getter(rename = "aggregation_bits_electra"))] pub aggregation_bits: BitList, pub signature: AggregateSignature, - pub index: u64, #[superstruct(only(Electra))] pub committee_bits: BitVector, } @@ -79,7 +78,6 @@ impl SplitAttestation { attesting_indices, aggregation_bits: attn.aggregation_bits, signature: attestation.signature().clone(), - index: data.index, }) } Attestation::Electra(attn) => { @@ -87,7 +85,6 @@ impl SplitAttestation { attesting_indices, aggregation_bits: attn.aggregation_bits, signature: attestation.signature().clone(), - index: data.index, committee_bits: attn.committee_bits, }) } @@ -182,18 +179,11 @@ impl CompactIndexedAttestation { ( CompactIndexedAttestation::Electra(this), CompactIndexedAttestation::Electra(other), - ) => this.aggregate(other), + ) => this.aggregate_same_committee(other), // TODO(electra) is a mix of electra and base compact indexed attestations an edge case we need to deal with? _ => (), } } - - pub fn committee_index(&self) -> u64 { - match self { - CompactIndexedAttestation::Base(att) => att.index, - CompactIndexedAttestation::Electra(att) => att.committee_index(), - } - } } impl CompactIndexedAttestationBase { @@ -225,14 +215,43 @@ impl CompactIndexedAttestationElectra { .is_zero() } - pub fn aggregate(&mut self, other: &Self) { + pub fn aggregate_same_committee(&mut self, other: &Self) { + // TODO(electra): remove assert in favour of Result + assert_eq!(self.committee_bits, other.committee_bits); + self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); + self.attesting_indices = self + .attesting_indices + .drain(..) + .merge(other.attesting_indices.iter().copied()) + .dedup() + .collect(); + self.signature.add_assign_aggregate(&other.signature); + } + + pub fn aggregate_with_disjoint_committees(&mut self, other: &Self) { + // TODO(electra): remove asserts or use Result + assert!( + self.committee_bits + .intersection(&other.committee_bits) + .is_zero(), + self.committee_bits, + other.committee_bits + ); + // The attestation being aggregated in must only have 1 committee bit set. + assert_eq!(other.committee_bits.num_set_bits(), 1); + // Check we are aggregating in increasing committee index order (so we can append + // aggregation bits). + assert!(self.committee_bits.highest_set_bit() < other.committee_bits.highest_set_bit()); + + self.committee_bits = self.committee_bits.union(&other.committee_bits); + self.aggregation_bits = + bitlist_extend(&self.aggregation_bits, &other.aggregation_bits).unwrap(); self.attesting_indices = self .attesting_indices .drain(..) .merge(other.attesting_indices.iter().copied()) .dedup() .collect(); - self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); self.signature.add_assign_aggregate(&other.signature); } @@ -249,6 +268,25 @@ impl CompactIndexedAttestationElectra { } } +// TODO(electra): upstream this or a more efficient implementation +fn bitlist_extend(list1: &BitList, list2: &BitList) -> Option> { + let new_length = list1.len() + list2.len(); + let mut list = BitList::::with_capacity(new_length).ok()?; + + // Copy bits from list1. + for (i, bit) in list1.iter().enumerate() { + list.set(i, bit).ok()?; + } + + // Copy bits from list2, starting from the end of list1. + let offset = list1.len(); + for (i, bit) in list2.iter().enumerate() { + list.set(offset + i, bit).ok()?; + } + + Some(list) +} + impl AttestationMap { pub fn insert(&mut self, attestation: Attestation, attesting_indices: Vec) { let SplitAttestation { @@ -279,102 +317,85 @@ impl AttestationMap { } } + /// Aggregate Electra attestations for the same attestation data signed by different + /// committees. + /// + /// Non-Electra attestations are left as-is. pub fn aggregate_across_committees(&mut self, checkpoint_key: CheckpointKey) { let Some(attestation_map) = self.checkpoint_map.get_mut(&checkpoint_key) else { return; }; - for (_, compact_indexed_attestations) in attestation_map.attestations.iter_mut() { + for compact_indexed_attestations in attestation_map.attestations.values_mut() { let unaggregated_attestations = std::mem::take(compact_indexed_attestations); let mut aggregated_attestations: Vec> = vec![]; // Aggregate the best attestations for each committee and leave the rest. - let mut best_attestations_by_committee: BTreeMap> = - BTreeMap::new(); + let mut best_attestations_by_committee: BTreeMap< + u64, + CompactIndexedAttestationElectra, + > = BTreeMap::new(); for committee_attestation in unaggregated_attestations { - // TODO(electra) - // compare to best attestations by committee - // could probably use `.entry` here + let mut electra_attestation = match committee_attestation { + CompactIndexedAttestation::Electra(att) + if att.committee_bits.num_set_bits() == 1 => + { + att + } + CompactIndexedAttestation::Electra(att) => { + // Aggregate already covers multiple committees, leave it as-is. + aggregated_attestations.push(CompactIndexedAttestation::Electra(att)); + continue; + } + CompactIndexedAttestation::Base(att) => { + // Leave as-is. + aggregated_attestations.push(CompactIndexedAttestation::Base(att)); + continue; + } + }; + let committee_index = electra_attestation.committee_index(); if let Some(existing_attestation) = - best_attestations_by_committee.get_mut(&committee_attestation.committee_index()) + best_attestations_by_committee.get_mut(&committee_index) { - // compare and swap, put the discarded one straight into - // `aggregated_attestations` in case we have room to pack it without - // cross-committee aggregation - if existing_attestation.should_aggregate(&committee_attestation) { - existing_attestation.aggregate(&committee_attestation); - - best_attestations_by_committee.insert( - committee_attestation.committee_index(), - committee_attestation, - ); - } else { - aggregated_attestations.push(committee_attestation); + // Search for the best (most aggregation bits) attestation for this committee + // index. + if electra_attestation.aggregation_bits.num_set_bits() + > existing_attestation.aggregation_bits.num_set_bits() + { + // New attestation is better than the previously known one for this + // committee. Replace it. + std::mem::swap(existing_attestation, &mut electra_attestation); } + // Put the inferior attestation into the list of aggregated attestations + // without performing any cross-committee aggregation. + aggregated_attestations + .push(CompactIndexedAttestation::Electra(electra_attestation)); } else { - best_attestations_by_committee.insert( - committee_attestation.committee_index(), - committee_attestation, - ); + // First attestation seen for this committee. Place it in the map + // provisionally. + best_attestations_by_committee.insert(committee_index, electra_attestation); } } - // TODO(electra): aggregate all the best attestations by committee - // (use btreemap sort order to get order by committee index) - aggregated_attestations.extend(Self::compute_on_chain_aggregate( - best_attestations_by_committee, - )); + if let Some(on_chain_aggregate) = + Self::compute_on_chain_aggregate(best_attestations_by_committee) + { + aggregated_attestations + .push(CompactIndexedAttestation::Electra(on_chain_aggregate)); + } *compact_indexed_attestations = aggregated_attestations; } } - // TODO(electra) unwraps in this function should be cleaned up - // also in general this could be a bit more elegant pub fn compute_on_chain_aggregate( - mut attestations_by_committee: BTreeMap>, - ) -> Vec> { - let mut aggregated_attestations = vec![]; - if let Some((_, on_chain_aggregate)) = attestations_by_committee.pop_first() { - match on_chain_aggregate { - CompactIndexedAttestation::Base(a) => { - aggregated_attestations.push(CompactIndexedAttestation::Base(a)); - aggregated_attestations.extend( - attestations_by_committee - .values() - .map(|a| { - CompactIndexedAttestation::Base(CompactIndexedAttestationBase { - attesting_indices: a.attesting_indices().clone(), - aggregation_bits: a.aggregation_bits_base().unwrap().clone(), - signature: a.signature().clone(), - index: *a.index(), - }) - }) - .collect::>>(), - ); - } - CompactIndexedAttestation::Electra(mut a) => { - for (_, attestation) in attestations_by_committee.iter_mut() { - let new_committee_bits = a - .committee_bits - .union(attestation.committee_bits().unwrap()); - a.aggregate(attestation.as_electra().unwrap()); - - a = CompactIndexedAttestationElectra { - attesting_indices: a.attesting_indices.clone(), - aggregation_bits: a.aggregation_bits.clone(), - signature: a.signature.clone(), - index: a.index, - committee_bits: new_committee_bits, - }; - } - - aggregated_attestations.push(CompactIndexedAttestation::Electra(a)); - } - } + mut attestations_by_committee: BTreeMap>, + ) -> Option> { + let (_, mut on_chain_aggregate) = attestations_by_committee.pop_first()?; + for (_, attestation) in attestations_by_committee { + on_chain_aggregate.aggregate_with_disjoint_committees(&attestation); } - - aggregated_attestations + Some(on_chain_aggregate) } /// Iterate all attestations matching the given `checkpoint_key`. diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 4f247e6bf20..daddbf76652 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -287,8 +287,10 @@ impl OperationPool { // TODO(electra): Work out how to do this more elegantly. This is a bit of a hack. let mut all_attestations = self.attestations.write(); - all_attestations.aggregate_across_committees(prev_epoch_key); - all_attestations.aggregate_across_committees(curr_epoch_key); + if fork_name >= ForkName::Electra { + all_attestations.aggregate_across_committees(prev_epoch_key); + all_attestations.aggregate_across_committees(curr_epoch_key); + } let all_attestations = parking_lot::RwLockWriteGuard::downgrade(all_attestations); From 16265ef455ca8c0f4c303bce89733bbd56646d1b Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 10 May 2024 12:44:18 +1000 Subject: [PATCH 048/227] Add comment to max cover optimisation --- beacon_node/operation_pool/src/attestation.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/beacon_node/operation_pool/src/attestation.rs b/beacon_node/operation_pool/src/attestation.rs index 207e2c65e45..91fd00a3979 100644 --- a/beacon_node/operation_pool/src/attestation.rs +++ b/beacon_node/operation_pool/src/attestation.rs @@ -144,6 +144,13 @@ impl<'a, E: EthSpec> MaxCover for AttMaxCover<'a, E> { /// because including two attestations on chain to satisfy different participation bits is /// impossible without the validator double voting. I.e. it is only suboptimal in the presence /// of slashable voting, which is rare. + /// + /// Post-Electra this optimisation is still OK. The `self.att.data.index` will always be 0 for + /// all Electra attestations, so when a new attestation is added to the solution, we will + /// remove its validators from all attestations at the same slot. It may happen that the + /// included attestation and the attestation being updated have no validators in common, in + /// which case the `retain` will be a no-op. We could consider optimising this in future by only + /// executing the `retain` when the `committee_bits` of the two attestations intersect. fn update_covering_set( &mut self, best_att: &AttestationRef<'a, E>, From 72548cb54e4a95f800e3b0591dba68ca0af7c6d5 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 10 May 2024 12:49:15 +1000 Subject: [PATCH 049/227] Fix assert --- beacon_node/operation_pool/src/attestation_storage.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index 4cc783e64f2..f06da2afb17 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -230,13 +230,10 @@ impl CompactIndexedAttestationElectra { pub fn aggregate_with_disjoint_committees(&mut self, other: &Self) { // TODO(electra): remove asserts or use Result - assert!( - self.committee_bits - .intersection(&other.committee_bits) - .is_zero(), - self.committee_bits, - other.committee_bits - ); + assert!(self + .committee_bits + .intersection(&other.committee_bits) + .is_zero(),); // The attestation being aggregated in must only have 1 committee bit set. assert_eq!(other.committee_bits.num_set_bits(), 1); // Check we are aggregating in increasing committee index order (so we can append From 08e045875f312f56356553332e6c43bc629e1c8a Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 7 May 2024 16:20:50 +1000 Subject: [PATCH 050/227] Electra epoch processing --- consensus/state_processing/src/epoch_cache.rs | 42 ++- .../src/per_epoch_processing/errors.rs | 1 + .../src/per_epoch_processing/single_pass.rs | 338 ++++++++++++++++-- consensus/types/src/validator.rs | 30 +- 4 files changed, 369 insertions(+), 42 deletions(-) diff --git a/consensus/state_processing/src/epoch_cache.rs b/consensus/state_processing/src/epoch_cache.rs index b2f2d85407e..3741f49c572 100644 --- a/consensus/state_processing/src/epoch_cache.rs +++ b/consensus/state_processing/src/epoch_cache.rs @@ -9,6 +9,7 @@ use types::{ActivationQueue, BeaconState, ChainSpec, EthSpec, ForkName, Hash256} pub struct PreEpochCache { epoch_key: EpochCacheKey, effective_balances: Vec, + total_active_balance: u64, } impl PreEpochCache { @@ -36,20 +37,51 @@ impl PreEpochCache { Ok(Self { epoch_key, effective_balances: Vec::with_capacity(state.validators().len()), + total_active_balance: 0, }) } - pub fn push_effective_balance(&mut self, effective_balance: u64) { - self.effective_balances.push(effective_balance); + pub fn update_effective_balance( + &mut self, + validator_index: usize, + effective_balance: u64, + is_active_next_epoch: bool, + ) -> Result<(), EpochCacheError> { + if validator_index == self.effective_balances.len() { + self.effective_balances.push(effective_balance); + if is_active_next_epoch { + self.total_active_balance + .safe_add_assign(effective_balance)?; + } + + Ok(()) + } else if let Some(existing_balance) = self.effective_balances.get_mut(validator_index) { + // Update total active balance for a late change in effective balance. This happens when + // processing consolidations. + if is_active_next_epoch { + self.total_active_balance + .safe_add_assign(effective_balance)?; + self.total_active_balance + .safe_sub_assign(*existing_balance)?; + } + *existing_balance = effective_balance; + Ok(()) + } else { + Err(EpochCacheError::ValidatorIndexOutOfBounds { validator_index }) + } + } + + pub fn get_total_active_balance(&self) -> u64 { + self.total_active_balance } pub fn into_epoch_cache( self, - total_active_balance: u64, activation_queue: ActivationQueue, spec: &ChainSpec, ) -> Result { let epoch = self.epoch_key.epoch; + let total_active_balance = self.total_active_balance; let sqrt_total_active_balance = SqrtTotalActiveBalance::new(total_active_balance); let base_reward_per_increment = BaseRewardPerIncrement::new(total_active_balance, spec)?; @@ -131,9 +163,9 @@ pub fn initialize_epoch_cache( decision_block_root, }, effective_balances, + total_active_balance, }; - *state.epoch_cache_mut() = - pre_epoch_cache.into_epoch_cache(total_active_balance, activation_queue, spec)?; + *state.epoch_cache_mut() = pre_epoch_cache.into_epoch_cache(activation_queue, spec)?; Ok(()) } diff --git a/consensus/state_processing/src/per_epoch_processing/errors.rs b/consensus/state_processing/src/per_epoch_processing/errors.rs index de481ec6767..7f60f4a6e01 100644 --- a/consensus/state_processing/src/per_epoch_processing/errors.rs +++ b/consensus/state_processing/src/per_epoch_processing/errors.rs @@ -25,6 +25,7 @@ pub enum EpochProcessingError { InvalidFlagIndex(usize), MilhouseError(milhouse::Error), EpochCache(EpochCacheError), + SinglePassMissingActivationQueue, } impl From for EpochProcessingError { diff --git a/consensus/state_processing/src/per_epoch_processing/single_pass.rs b/consensus/state_processing/src/per_epoch_processing/single_pass.rs index a9629e73e40..3d72f0cdd86 100644 --- a/consensus/state_processing/src/per_epoch_processing/single_pass.rs +++ b/consensus/state_processing/src/per_epoch_processing/single_pass.rs @@ -1,20 +1,24 @@ use crate::{ - common::update_progressive_balances_cache::initialize_progressive_balances_cache, + common::{ + decrease_balance, increase_balance, + update_progressive_balances_cache::initialize_progressive_balances_cache, + }, epoch_cache::{initialize_epoch_cache, PreEpochCache}, per_epoch_processing::{Delta, Error, ParticipationEpochSummary}, }; use itertools::izip; use safe_arith::{SafeArith, SafeArithIter}; use std::cmp::{max, min}; -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use types::{ consts::altair::{ NUM_FLAG_INDICES, PARTICIPATION_FLAG_WEIGHTS, TIMELY_HEAD_FLAG_INDEX, TIMELY_TARGET_FLAG_INDEX, WEIGHT_DENOMINATOR, }, milhouse::Cow, - ActivationQueue, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ExitCache, ForkName, - ParticipationFlags, ProgressiveBalancesCache, RelativeEpoch, Unsigned, Validator, + ActivationQueue, BeaconState, BeaconStateError, ChainSpec, Checkpoint, Epoch, EthSpec, + ExitCache, ForkName, List, ParticipationFlags, ProgressiveBalancesCache, RelativeEpoch, + Unsigned, Validator, }; pub struct SinglePassConfig { @@ -22,6 +26,8 @@ pub struct SinglePassConfig { pub rewards_and_penalties: bool, pub registry_updates: bool, pub slashings: bool, + pub pending_balance_deposits: bool, + pub pending_consolidations: bool, pub effective_balance_updates: bool, } @@ -38,6 +44,8 @@ impl SinglePassConfig { rewards_and_penalties: true, registry_updates: true, slashings: true, + pending_balance_deposits: true, + pending_consolidations: true, effective_balance_updates: true, } } @@ -48,6 +56,8 @@ impl SinglePassConfig { rewards_and_penalties: false, registry_updates: false, slashings: false, + pending_balance_deposits: false, + pending_consolidations: false, effective_balance_updates: false, } } @@ -57,6 +67,7 @@ impl SinglePassConfig { struct StateContext { current_epoch: Epoch, next_epoch: Epoch, + finalized_checkpoint: Checkpoint, is_in_inactivity_leak: bool, total_active_balance: u64, churn_limit: u64, @@ -73,6 +84,15 @@ struct SlashingsContext { target_withdrawable_epoch: Epoch, } +struct PendingBalanceDepositsContext { + /// The value to set `next_deposit_index` to *after* processing completes. + next_deposit_index: usize, + /// The value to set `deposit_balance_to_consume` to *after* processing completes. + deposit_balance_to_consume: u64, + /// Total balance increases for each validator due to pending balance deposits. + validator_deposits_to_process: HashMap, +} + struct EffectiveBalancesContext { downward_threshold: u64, upward_threshold: u64, @@ -129,6 +149,7 @@ pub fn process_epoch_single_pass( let state_ctxt = &StateContext { current_epoch, next_epoch, + finalized_checkpoint, is_in_inactivity_leak, total_active_balance, churn_limit, @@ -139,6 +160,13 @@ pub fn process_epoch_single_pass( let slashings_ctxt = &SlashingsContext::new(state, state_ctxt, spec)?; let mut next_epoch_cache = PreEpochCache::new_for_next_epoch(state)?; + let pending_balance_deposits_ctxt = + if fork_name >= ForkName::Electra && conf.pending_balance_deposits { + Some(PendingBalanceDepositsContext::new(state, spec)?) + } else { + None + }; + // Split the state into several disjoint mutable borrows. let ( validators, @@ -165,12 +193,19 @@ pub fn process_epoch_single_pass( // Compute shared values required for different parts of epoch processing. let rewards_ctxt = &RewardsAndPenaltiesContext::new(progressive_balances, state_ctxt, spec)?; - let activation_queue = &epoch_cache - .activation_queue()? - .get_validators_eligible_for_activation( - finalized_checkpoint.epoch, - activation_churn_limit as usize, - ); + + let mut activation_queues = if fork_name < ForkName::Electra { + let activation_queue = epoch_cache + .activation_queue()? + .get_validators_eligible_for_activation( + finalized_checkpoint.epoch, + activation_churn_limit as usize, + ); + let next_epoch_activation_queue = ActivationQueue::default(); + Some((activation_queue, next_epoch_activation_queue)) + } else { + None + }; let effective_balances_ctxt = &EffectiveBalancesContext::new(spec)?; // Iterate over the validators and related fields in one pass. @@ -178,10 +213,6 @@ pub fn process_epoch_single_pass( let mut balances_iter = balances.iter_cow(); let mut inactivity_scores_iter = inactivity_scores.iter_cow(); - // Values computed for the next epoch transition. - let mut next_epoch_total_active_balance = 0; - let mut next_epoch_activation_queue = ActivationQueue::default(); - for (index, &previous_epoch_participation, ¤t_epoch_participation) in izip!( 0..num_validators, previous_epoch_participation.iter(), @@ -246,12 +277,14 @@ pub fn process_epoch_single_pass( // `process_registry_updates` if conf.registry_updates { + let activation_queue_refs = activation_queues + .as_mut() + .map(|(current_queue, next_queue)| (&*current_queue, next_queue)); process_single_registry_update( &mut validator, validator_info, exit_cache, - activation_queue, - &mut next_epoch_activation_queue, + activation_queue_refs, state_ctxt, spec, )?; @@ -262,13 +295,22 @@ pub fn process_epoch_single_pass( process_single_slashing(&mut balance, &validator, slashings_ctxt, state_ctxt, spec)?; } + // `process_pending_balance_deposits` + if let Some(pending_balance_deposits_ctxt) = &pending_balance_deposits_ctxt { + process_pending_balance_deposits_for_validator( + &mut balance, + validator_info, + pending_balance_deposits_ctxt, + )?; + } + // `process_effective_balance_updates` if conf.effective_balance_updates { process_single_effective_balance_update( + validator_info.index, *balance, &mut validator, - validator_info, - &mut next_epoch_total_active_balance, + validator_info.current_epoch_participation, &mut next_epoch_cache, progressive_balances, effective_balances_ctxt, @@ -278,15 +320,45 @@ pub fn process_epoch_single_pass( } } - if conf.effective_balance_updates { - state.set_total_active_balance(next_epoch, next_epoch_total_active_balance, spec); - *state.epoch_cache_mut() = next_epoch_cache.into_epoch_cache( - next_epoch_total_active_balance, - next_epoch_activation_queue, + // Finish processing pending balance deposits if relevant. + // + // This *could* be reordered after `process_pending_consolidations` which pushes only to the end + // of the `pending_balance_deposits` list. But we may as well preserve the write ordering used + // by the spec and do this first. + if let Some(ctxt) = pending_balance_deposits_ctxt { + let new_pending_balance_deposits = List::try_from_iter( + state + .pending_balance_deposits()? + .iter_from(ctxt.next_deposit_index)? + .cloned(), + )?; + *state.pending_balance_deposits_mut()? = new_pending_balance_deposits; + *state.deposit_balance_to_consume_mut()? = ctxt.deposit_balance_to_consume; + } + + // Process consolidations outside the single-pass loop, as they depend on balances for multiple + // validators and cannot be computed accurately inside the loop. + if fork_name >= ForkName::Electra && conf.pending_consolidations { + process_pending_consolidations( + state, + &mut next_epoch_cache, + effective_balances_ctxt, + state_ctxt, spec, )?; } + // Finally, finish updating effective balance caches. We need this to happen *after* processing + // of pending consolidations, which recomputes some effective balances. + if conf.effective_balance_updates { + let next_epoch_total_active_balance = next_epoch_cache.get_total_active_balance(); + state.set_total_active_balance(next_epoch, next_epoch_total_active_balance, spec); + let next_epoch_activation_queue = + activation_queues.map_or_else(ActivationQueue::default, |(_, queue)| queue); + *state.epoch_cache_mut() = + next_epoch_cache.into_epoch_cache(next_epoch_activation_queue, spec)?; + } + Ok(summary) } @@ -456,6 +528,31 @@ impl RewardsAndPenaltiesContext { } fn process_single_registry_update( + validator: &mut Cow, + validator_info: &ValidatorInfo, + exit_cache: &mut ExitCache, + activation_queues: Option<(&BTreeSet, &mut ActivationQueue)>, + state_ctxt: &StateContext, + spec: &ChainSpec, +) -> Result<(), Error> { + if state_ctxt.fork_name < ForkName::Electra { + let (activation_queue, next_epoch_activation_queue) = + activation_queues.ok_or(Error::SinglePassMissingActivationQueue)?; + process_single_registry_update_pre_electra( + validator, + validator_info, + exit_cache, + activation_queue, + next_epoch_activation_queue, + state_ctxt, + spec, + ) + } else { + process_single_registry_update_post_electra(validator, exit_cache, state_ctxt, spec) + } +} + +fn process_single_registry_update_pre_electra( validator: &mut Cow, validator_info: &ValidatorInfo, exit_cache: &mut ExitCache, @@ -491,6 +588,35 @@ fn process_single_registry_update( Ok(()) } +fn process_single_registry_update_post_electra( + validator: &mut Cow, + exit_cache: &mut ExitCache, + state_ctxt: &StateContext, + spec: &ChainSpec, +) -> Result<(), Error> { + let current_epoch = state_ctxt.current_epoch; + + if validator.is_eligible_for_activation_queue(spec, state_ctxt.fork_name) { + validator.make_mut()?.activation_eligibility_epoch = current_epoch.safe_add(1)?; + } + + if validator.is_active_at(current_epoch) && validator.effective_balance <= spec.ejection_balance + { + // TODO(electra): make sure initiate_validator_exit is updated + initiate_validator_exit(validator, exit_cache, state_ctxt, spec)?; + } + + if validator.is_eligible_for_activation_with_finalized_checkpoint( + &state_ctxt.finalized_checkpoint, + spec, + ) { + validator.make_mut()?.activation_epoch = + spec.compute_activation_exit_epoch(current_epoch)?; + } + + Ok(()) +} + fn initiate_validator_exit( validator: &mut Cow, exit_cache: &mut ExitCache, @@ -568,6 +694,146 @@ fn process_single_slashing( Ok(()) } +impl PendingBalanceDepositsContext { + fn new(state: &BeaconState, spec: &ChainSpec) -> Result { + let available_for_processing = state + .deposit_balance_to_consume()? + .safe_add(state.get_activation_exit_churn_limit(spec)?)?; + let mut processed_amount = 0; + let mut next_deposit_index = 0; + let mut validator_deposits_to_process = HashMap::new(); + + let pending_balance_deposits = state.pending_balance_deposits()?; + + for deposit in pending_balance_deposits.iter() { + if processed_amount.safe_add(deposit.amount)? > available_for_processing { + break; + } + validator_deposits_to_process + .entry(deposit.index as usize) + .or_insert(0) + .safe_add_assign(deposit.amount)?; + + processed_amount.safe_add_assign(deposit.amount)?; + next_deposit_index.safe_add_assign(1)?; + } + + let deposit_balance_to_consume = if next_deposit_index == pending_balance_deposits.len() { + 0 + } else { + available_for_processing.safe_sub(processed_amount)? + }; + + Ok(Self { + next_deposit_index, + deposit_balance_to_consume, + validator_deposits_to_process, + }) + } +} + +fn process_pending_balance_deposits_for_validator( + balance: &mut Cow, + validator_info: &ValidatorInfo, + pending_balance_deposits_ctxt: &PendingBalanceDepositsContext, +) -> Result<(), Error> { + if let Some(deposit_amount) = pending_balance_deposits_ctxt + .validator_deposits_to_process + .get(&validator_info.index) + { + balance.make_mut()?.safe_add_assign(*deposit_amount)?; + } + Ok(()) +} + +/// We process pending consolidations after all of single-pass epoch processing, and then patch up +/// the effective balances for affected validators. +/// +/// This is safe because processing consolidations does not depend on the `effective_balance`. +fn process_pending_consolidations( + state: &mut BeaconState, + next_epoch_cache: &mut PreEpochCache, + effective_balances_ctxt: &EffectiveBalancesContext, + state_ctxt: &StateContext, + spec: &ChainSpec, +) -> Result<(), Error> { + let mut next_pending_consolidation: usize = 0; + let current_epoch = state.current_epoch(); + let pending_consolidations = state.pending_consolidations()?.clone(); + + let mut affected_validators = BTreeSet::new(); + + for pending_consolidation in &pending_consolidations { + let source_index = pending_consolidation.source_index as usize; + let target_index = pending_consolidation.target_index as usize; + let source_validator = state.get_validator(source_index)?; + if source_validator.slashed { + next_pending_consolidation.safe_add_assign(1)?; + continue; + } + if source_validator.withdrawable_epoch > current_epoch { + break; + } + + // Calculate the active balance while we have the source validator loaded. This is a safe + // reordering. + let source_balance = *state + .balances() + .get(source_index) + .ok_or(BeaconStateError::UnknownValidator(source_index))?; + let active_balance = + source_validator.get_active_balance(source_balance, spec, state_ctxt.fork_name); + + // Churn any target excess active balance of target and raise its max. + state.switch_to_compounding_validator(target_index, spec)?; + + // Move active balance to target. Excess balance is withdrawable. + decrease_balance(state, source_index, active_balance)?; + increase_balance(state, target_index, active_balance)?; + + affected_validators.insert(source_index); + affected_validators.insert(target_index); + + next_pending_consolidation.safe_add_assign(1)?; + } + + let new_pending_consolidations = List::try_from_iter( + state + .pending_consolidations()? + .iter_from(next_pending_consolidation)? + .cloned(), + )?; + *state.pending_consolidations_mut()? = new_pending_consolidations; + + // Re-process effective balance updates for validators affected by consolidations. + let (validators, balances, _, current_epoch_participation, _, progressive_balances, _, _) = + state.mutable_validator_fields()?; + for validator_index in affected_validators { + let balance = *balances + .get(validator_index) + .ok_or(BeaconStateError::UnknownValidator(validator_index))?; + let mut validator = validators + .get_cow(validator_index) + .ok_or(BeaconStateError::UnknownValidator(validator_index))?; + let validator_current_epoch_participation = *current_epoch_participation + .get(validator_index) + .ok_or(BeaconStateError::UnknownValidator(validator_index))?; + + process_single_effective_balance_update( + validator_index, + balance, + &mut validator, + validator_current_epoch_participation, + next_epoch_cache, + progressive_balances, + effective_balances_ctxt, + state_ctxt, + spec, + )?; + } + Ok(()) +} + impl EffectiveBalancesContext { fn new(spec: &ChainSpec) -> Result { let hysteresis_increment = spec @@ -584,18 +850,24 @@ impl EffectiveBalancesContext { } } +/// This function abstracts over phase0 and Electra effective balance processing. #[allow(clippy::too_many_arguments)] fn process_single_effective_balance_update( + validator_index: usize, balance: u64, validator: &mut Cow, - validator_info: &ValidatorInfo, - next_epoch_total_active_balance: &mut u64, + validator_current_epoch_participation: ParticipationFlags, next_epoch_cache: &mut PreEpochCache, progressive_balances: &mut ProgressiveBalancesCache, eb_ctxt: &EffectiveBalancesContext, state_ctxt: &StateContext, spec: &ChainSpec, ) -> Result<(), Error> { + // Use the higher effective balance limit if post-Electra and compounding withdrawal credentials + // are set. + let effective_balance_limit = + validator.get_validator_max_effective_balance(spec, state_ctxt.fork_name); + let old_effective_balance = validator.effective_balance; let new_effective_balance = if balance.safe_add(eb_ctxt.downward_threshold)? < validator.effective_balance @@ -606,15 +878,13 @@ fn process_single_effective_balance_update( { min( balance.safe_sub(balance.safe_rem(spec.effective_balance_increment)?)?, - spec.max_effective_balance, + effective_balance_limit, ) } else { validator.effective_balance }; - if validator.is_active_at(state_ctxt.next_epoch) { - next_epoch_total_active_balance.safe_add_assign(new_effective_balance)?; - } + let is_active_next_epoch = validator.is_active_at(state_ctxt.next_epoch); if new_effective_balance != old_effective_balance { validator.make_mut()?.effective_balance = new_effective_balance; @@ -623,14 +893,18 @@ fn process_single_effective_balance_update( // previous epoch once the epoch transition completes. progressive_balances.on_effective_balance_change( validator.slashed, - validator_info.current_epoch_participation, + validator_current_epoch_participation, old_effective_balance, new_effective_balance, )?; } - // Caching: update next epoch effective balances. - next_epoch_cache.push_effective_balance(new_effective_balance); + // Caching: update next epoch effective balances and total active balance. + next_epoch_cache.update_effective_balance( + validator_index, + new_effective_balance, + is_active_next_epoch, + )?; Ok(()) } diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index 86243083f06..82d0ad311bb 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -1,6 +1,6 @@ use crate::{ - test_utils::TestRandom, Address, BeaconState, ChainSpec, Epoch, EthSpec, ForkName, Hash256, - PublicKeyBytes, + test_utils::TestRandom, Address, BeaconState, ChainSpec, Checkpoint, Epoch, EthSpec, ForkName, + Hash256, PublicKeyBytes, }; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -87,15 +87,25 @@ impl Validator { } /// Returns `true` if the validator is eligible to be activated. - /// - /// Spec v0.12.1 pub fn is_eligible_for_activation( &self, state: &BeaconState, spec: &ChainSpec, + ) -> bool { + self.is_eligible_for_activation_with_finalized_checkpoint( + &state.finalized_checkpoint(), + spec, + ) + } + + /// Returns `true` if the validator is eligible to be activated. + pub fn is_eligible_for_activation_with_finalized_checkpoint( + &self, + finalized_checkpoint: &Checkpoint, + spec: &ChainSpec, ) -> bool { // Placement in queue is finalized - self.activation_eligibility_epoch <= state.finalized_checkpoint().epoch + self.activation_eligibility_epoch <= finalized_checkpoint.epoch // Has not yet been activated && self.activation_epoch == spec.far_future_epoch } @@ -267,6 +277,16 @@ impl Validator { spec.max_effective_balance } } + + pub fn get_active_balance( + &self, + validator_balance: u64, + spec: &ChainSpec, + current_fork: ForkName, + ) -> u64 { + let max_effective_balance = self.get_validator_max_effective_balance(spec, current_fork); + std::cmp::min(validator_balance, max_effective_balance) + } } impl Default for Validator { From ba02ffc162be74a301a2bcbad70d7efdf3b5784c Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 10 May 2024 09:52:51 -0400 Subject: [PATCH 051/227] add deposit limit for old deposit queue --- beacon_node/beacon_chain/src/eth1_chain.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index 31297244e3e..229b3fd5251 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -546,12 +546,20 @@ impl Eth1ChainBackend for CachingEth1Backend { state.eth1_data().deposit_count }; - match deposit_index.cmp(&deposit_count) { + // [New in Electra:EIP6110] + let deposit_index_limit = + if let Ok(deposit_receipts_start_index) = state.deposit_receipts_start_index() { + std::cmp::min(deposit_count, deposit_receipts_start_index) + } else { + deposit_count + }; + + match deposit_index.cmp(&deposit_index_limit) { Ordering::Greater => Err(Error::DepositIndexTooHigh), Ordering::Equal => Ok(vec![]), Ordering::Less => { let next = deposit_index; - let last = std::cmp::min(deposit_count, next + E::MaxDeposits::to_u64()); + let last = std::cmp::min(deposit_index_limit, next + E::MaxDeposits::to_u64()); self.core .deposits() From 89e4de90d5771ea4b56473f010ff6e5c9f32a76e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 10 May 2024 12:09:39 -0400 Subject: [PATCH 052/227] don't fail on empty consolidations --- .../src/per_block_processing/process_operations.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index aba184b8043..0fbb82ed1d9 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -653,6 +653,10 @@ pub fn process_consolidations( verify_signatures: VerifySignatures, spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { + if consolidations.is_empty() { + return Ok(()); + } + // If the pending consolidations queue is full, no consolidations are allowed in the block let pending_consolidations = state.pending_consolidations()?.len(); let pending_consolidations_limit = E::pending_consolidations_limit(); From e1dcfb6960ae74f14302ecfbb5075e0963066355 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 10 May 2024 21:15:21 -0400 Subject: [PATCH 053/227] update committee offset --- .../state_processing/src/common/get_attesting_indices.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 9848840e96d..33249c49417 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -106,7 +106,7 @@ pub mod attesting_indices_electra { let committee_indices = get_committee_indices::(committee_bits); - let committee_offset = 0; + let mut committee_offset = 0; let committees_map: HashMap = committees .iter() @@ -138,7 +138,7 @@ pub mod attesting_indices_electra { output.extend(committee_attesters); - committee_offset.safe_add(beacon_committee.committee.len())?; + committee_offset.safe_add_assign(beacon_committee.committee.len())?; } else { return Err(Error::NoCommitteeFound(index)); } From 40c4c00097c2e9cc1d06bfec9a6d8ec374140d5b Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 10 May 2024 21:15:21 -0400 Subject: [PATCH 054/227] update committee offset --- .../state_processing/src/common/get_attesting_indices.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 9848840e96d..33249c49417 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -106,7 +106,7 @@ pub mod attesting_indices_electra { let committee_indices = get_committee_indices::(committee_bits); - let committee_offset = 0; + let mut committee_offset = 0; let committees_map: HashMap = committees .iter() @@ -138,7 +138,7 @@ pub mod attesting_indices_electra { output.extend(committee_attesters); - committee_offset.safe_add(beacon_committee.committee.len())?; + committee_offset.safe_add_assign(beacon_committee.committee.len())?; } else { return Err(Error::NoCommitteeFound(index)); } From 5364ba53fa9a9f4bc4955f621bc61718f031a6de Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 10 May 2024 21:15:21 -0400 Subject: [PATCH 055/227] update committee offset --- .../state_processing/src/common/get_attesting_indices.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 9848840e96d..33249c49417 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -106,7 +106,7 @@ pub mod attesting_indices_electra { let committee_indices = get_committee_indices::(committee_bits); - let committee_offset = 0; + let mut committee_offset = 0; let committees_map: HashMap = committees .iter() @@ -138,7 +138,7 @@ pub mod attesting_indices_electra { output.extend(committee_attesters); - committee_offset.safe_add(beacon_committee.committee.len())?; + committee_offset.safe_add_assign(beacon_committee.committee.len())?; } else { return Err(Error::NoCommitteeFound(index)); } From 9a22eb8698f8373c1bb002027edb41b300043bd4 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 10 May 2024 21:15:21 -0400 Subject: [PATCH 056/227] update committee offset --- .../state_processing/src/common/get_attesting_indices.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 9848840e96d..33249c49417 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -106,7 +106,7 @@ pub mod attesting_indices_electra { let committee_indices = get_committee_indices::(committee_bits); - let committee_offset = 0; + let mut committee_offset = 0; let committees_map: HashMap = committees .iter() @@ -138,7 +138,7 @@ pub mod attesting_indices_electra { output.extend(committee_attesters); - committee_offset.safe_add(beacon_committee.committee.len())?; + committee_offset.safe_add_assign(beacon_committee.committee.len())?; } else { return Err(Error::NoCommitteeFound(index)); } From 879966e535601c83c6af0fc451388aee3cf0022c Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 10 May 2024 21:15:21 -0400 Subject: [PATCH 057/227] update committee offset --- .../state_processing/src/common/get_attesting_indices.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 1c59edf20e7..803686cbbde 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -106,7 +106,7 @@ pub mod attesting_indices_electra { let committee_indices = get_committee_indices::(committee_bits); - let committee_offset = 0; + let mut committee_offset = 0; let committees_map: HashMap = committees .iter() @@ -138,7 +138,7 @@ pub mod attesting_indices_electra { output.extend(committee_attesters); - committee_offset.safe_add(beacon_committee.committee.len())?; + committee_offset.safe_add_assign(beacon_committee.committee.len())?; } else { return Err(Error::NoCommitteeFound(index)); } From bf167c3c46f93f2a118ca71bc7de27cdb1e8a8f3 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 10 May 2024 21:15:21 -0400 Subject: [PATCH 058/227] update committee offset --- .../state_processing/src/common/get_attesting_indices.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 1c59edf20e7..803686cbbde 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -106,7 +106,7 @@ pub mod attesting_indices_electra { let committee_indices = get_committee_indices::(committee_bits); - let committee_offset = 0; + let mut committee_offset = 0; let committees_map: HashMap = committees .iter() @@ -138,7 +138,7 @@ pub mod attesting_indices_electra { output.extend(committee_attesters); - committee_offset.safe_add(beacon_committee.committee.len())?; + committee_offset.safe_add_assign(beacon_committee.committee.len())?; } else { return Err(Error::NoCommitteeFound(index)); } From 817d4235ae20c2f37feba1d6cbc774157a8b8ff1 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 05:00:11 -0400 Subject: [PATCH 059/227] only increment deposit index on state for old deposit flow --- beacon_node/beacon_chain/src/eth1_chain.rs | 9 +++++++++ .../genesis/src/eth1_genesis_service.rs | 2 +- consensus/state_processing/src/genesis.rs | 2 +- .../process_operations.rs | 18 +++++++++++++++--- consensus/types/src/beacon_state.rs | 2 ++ testing/ef_tests/src/cases/operations.rs | 4 +++- testing/ef_tests/src/cases/sanity_blocks.rs | 19 ++++++++++++------- 7 files changed, 43 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index 229b3fd5251..a83c5eb135c 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -549,6 +549,7 @@ impl Eth1ChainBackend for CachingEth1Backend { // [New in Electra:EIP6110] let deposit_index_limit = if let Ok(deposit_receipts_start_index) = state.deposit_receipts_start_index() { + dbg!("deposit_receipts_start_index", deposit_receipts_start_index); std::cmp::min(deposit_count, deposit_receipts_start_index) } else { deposit_count @@ -561,6 +562,14 @@ impl Eth1ChainBackend for CachingEth1Backend { let next = deposit_index; let last = std::cmp::min(deposit_index_limit, next + E::MaxDeposits::to_u64()); + dbg!( + next, + last, + deposit_count, + deposit_index_limit, + E::MaxDeposits::to_u64() + ); + self.core .deposits() .read() diff --git a/beacon_node/genesis/src/eth1_genesis_service.rs b/beacon_node/genesis/src/eth1_genesis_service.rs index 3e91504951c..70157050278 100644 --- a/beacon_node/genesis/src/eth1_genesis_service.rs +++ b/beacon_node/genesis/src/eth1_genesis_service.rs @@ -439,7 +439,7 @@ impl Eth1GenesisService { None }; - apply_deposit(&mut state, data, proof, spec) + apply_deposit(&mut state, data, proof, true, spec) .map_err(|e| format!("Error whilst processing deposit: {:?}", e)) })?; diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 38006b83cac..c73417077ae 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -38,7 +38,7 @@ pub fn initialize_beacon_state_from_eth1( .map_err(BlockProcessingError::MerkleTreeError)?; state.eth1_data_mut().deposit_root = deposit_tree.root(); let Deposit { proof, data } = deposit; - apply_deposit(&mut state, data, Some(proof), spec)?; + apply_deposit(&mut state, data, Some(proof), true, spec)?; } process_activations(&mut state, spec)?; diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 0fbb82ed1d9..9b9408f4d19 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -417,7 +417,7 @@ pub fn process_deposits( // Update the state in series. for deposit in deposits { - apply_deposit(state, deposit.data.clone(), None, spec)?; + apply_deposit(state, deposit.data.clone(), None, true, spec)?; } Ok(()) @@ -428,6 +428,7 @@ pub fn apply_deposit( state: &mut BeaconState, deposit_data: DepositData, proof: Option>, + increment_deposit_index: bool, spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { let deposit_index = state.eth1_deposit_index() as usize; @@ -440,7 +441,9 @@ pub fn apply_deposit( .map_err(|e| e.into_with_index(deposit_index))?; } - state.eth1_deposit_index_mut().safe_add_assign(1)?; + if increment_deposit_index { + state.eth1_deposit_index_mut().safe_add_assign(1)?; + } // Get an `Option` where `u64` is the validator index if this deposit public key // already exists in the beacon_state. @@ -459,6 +462,12 @@ pub fn apply_deposit( .get(index as usize) .ok_or(BeaconStateError::UnknownValidator(index as usize))?; + dbg!(is_compounding_withdrawal_credential( + deposit_data.withdrawal_credentials, + spec + )); + dbg!(validator.has_eth1_withdrawal_credential(spec)); + dbg!(is_valid_deposit_signature(&deposit_data, spec).is_ok()); if is_compounding_withdrawal_credential(deposit_data.withdrawal_credentials, spec) && validator.has_eth1_withdrawal_credential(spec) && is_valid_deposit_signature(&deposit_data, spec).is_ok() @@ -491,6 +500,8 @@ pub fn apply_deposit( amount, ) }; + dbg!(effective_balance, state_balance); + dbg!(&deposit_data.pubkey, &deposit_data.withdrawal_credentials); // Create a new validator. let validator = Validator { pubkey: deposit_data.pubkey, @@ -518,6 +529,7 @@ pub fn apply_deposit( // [New in Electra:EIP7251] if let Ok(pending_balance_deposits) = state.pending_balance_deposits_mut() { + dbg!(new_validator_index, amount); pending_balance_deposits.push(PendingBalanceDeposit { index: new_validator_index as u64, amount, @@ -641,7 +653,7 @@ pub fn process_deposit_receipts( amount: receipt.amount, signature: receipt.signature.clone().into(), }; - apply_deposit(state, deposit_data, None, spec)? + apply_deposit(state, deposit_data, None, false, spec)? } Ok(()) diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 6b4bfd29cb5..9a1fe609889 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -2151,6 +2151,7 @@ impl BeaconState { if *balance > spec.min_activation_balance { let excess_balance = balance.safe_sub(spec.min_activation_balance)?; *balance = spec.min_activation_balance; + dbg!(validator_index, excess_balance); self.pending_balance_deposits_mut()? .push(PendingBalanceDeposit { index: validator_index as u64, @@ -2200,6 +2201,7 @@ impl BeaconState { if validator.has_eth1_withdrawal_credential(spec) { validator.withdrawal_credentials.as_fixed_bytes_mut()[0] = spec.compounding_withdrawal_prefix_byte; + dbg!(validator.withdrawal_credentials); self.queue_excess_active_balance(validator_index, spec)?; } Ok(()) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index f7adcbd75ee..3f862903615 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -171,7 +171,9 @@ impl Operation for Deposit { spec: &ChainSpec, _: &Operations, ) -> Result<(), BlockProcessingError> { - process_deposits(state, &[self.clone()], spec) + let res = process_deposits(state, &[self.clone()], spec); + dbg!(serde_json::to_string(state).unwrap()); + res } } diff --git a/testing/ef_tests/src/cases/sanity_blocks.rs b/testing/ef_tests/src/cases/sanity_blocks.rs index 91bb995cc43..3232ddee04e 100644 --- a/testing/ef_tests/src/cases/sanity_blocks.rs +++ b/testing/ef_tests/src/cases/sanity_blocks.rs @@ -60,6 +60,9 @@ impl Case for SanityBlocks { } fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> { + if _case_index != 74 || fork_name != ForkName::Electra { + return Ok(()); + } self.metadata.bls_setting.unwrap_or_default().check()?; let mut bulk_state = self.pre.clone(); @@ -111,13 +114,15 @@ impl Case for SanityBlocks { spec, )?; - if block.state_root() == bulk_state.update_tree_hash_cache().unwrap() - && block.state_root() == indiv_state.update_tree_hash_cache().unwrap() - { - Ok(()) - } else { - Err(BlockProcessingError::StateRootMismatch) - } + // if block.state_root() == bulk_state.update_tree_hash_cache().unwrap() + // && block.state_root() == indiv_state.update_tree_hash_cache().unwrap() + // { + // Ok(()) + // } else { + // Err(BlockProcessingError::StateRootMismatch) + // } + + Ok::<_, BlockProcessingError>(()) }) .map(|_| (bulk_state, indiv_state)); From 5f09453287708902bde141c3f06514a69743abf0 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 05:00:47 -0400 Subject: [PATCH 060/227] Revert "only increment deposit index on state for old deposit flow" This reverts commit 817d4235ae20c2f37feba1d6cbc774157a8b8ff1. --- beacon_node/beacon_chain/src/eth1_chain.rs | 9 --------- .../genesis/src/eth1_genesis_service.rs | 2 +- consensus/state_processing/src/genesis.rs | 2 +- .../process_operations.rs | 18 +++--------------- consensus/types/src/beacon_state.rs | 2 -- testing/ef_tests/src/cases/operations.rs | 4 +--- testing/ef_tests/src/cases/sanity_blocks.rs | 19 +++++++------------ 7 files changed, 13 insertions(+), 43 deletions(-) diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index a83c5eb135c..229b3fd5251 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -549,7 +549,6 @@ impl Eth1ChainBackend for CachingEth1Backend { // [New in Electra:EIP6110] let deposit_index_limit = if let Ok(deposit_receipts_start_index) = state.deposit_receipts_start_index() { - dbg!("deposit_receipts_start_index", deposit_receipts_start_index); std::cmp::min(deposit_count, deposit_receipts_start_index) } else { deposit_count @@ -562,14 +561,6 @@ impl Eth1ChainBackend for CachingEth1Backend { let next = deposit_index; let last = std::cmp::min(deposit_index_limit, next + E::MaxDeposits::to_u64()); - dbg!( - next, - last, - deposit_count, - deposit_index_limit, - E::MaxDeposits::to_u64() - ); - self.core .deposits() .read() diff --git a/beacon_node/genesis/src/eth1_genesis_service.rs b/beacon_node/genesis/src/eth1_genesis_service.rs index 70157050278..3e91504951c 100644 --- a/beacon_node/genesis/src/eth1_genesis_service.rs +++ b/beacon_node/genesis/src/eth1_genesis_service.rs @@ -439,7 +439,7 @@ impl Eth1GenesisService { None }; - apply_deposit(&mut state, data, proof, true, spec) + apply_deposit(&mut state, data, proof, spec) .map_err(|e| format!("Error whilst processing deposit: {:?}", e)) })?; diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index c73417077ae..38006b83cac 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -38,7 +38,7 @@ pub fn initialize_beacon_state_from_eth1( .map_err(BlockProcessingError::MerkleTreeError)?; state.eth1_data_mut().deposit_root = deposit_tree.root(); let Deposit { proof, data } = deposit; - apply_deposit(&mut state, data, Some(proof), true, spec)?; + apply_deposit(&mut state, data, Some(proof), spec)?; } process_activations(&mut state, spec)?; diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 9b9408f4d19..0fbb82ed1d9 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -417,7 +417,7 @@ pub fn process_deposits( // Update the state in series. for deposit in deposits { - apply_deposit(state, deposit.data.clone(), None, true, spec)?; + apply_deposit(state, deposit.data.clone(), None, spec)?; } Ok(()) @@ -428,7 +428,6 @@ pub fn apply_deposit( state: &mut BeaconState, deposit_data: DepositData, proof: Option>, - increment_deposit_index: bool, spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { let deposit_index = state.eth1_deposit_index() as usize; @@ -441,9 +440,7 @@ pub fn apply_deposit( .map_err(|e| e.into_with_index(deposit_index))?; } - if increment_deposit_index { - state.eth1_deposit_index_mut().safe_add_assign(1)?; - } + state.eth1_deposit_index_mut().safe_add_assign(1)?; // Get an `Option` where `u64` is the validator index if this deposit public key // already exists in the beacon_state. @@ -462,12 +459,6 @@ pub fn apply_deposit( .get(index as usize) .ok_or(BeaconStateError::UnknownValidator(index as usize))?; - dbg!(is_compounding_withdrawal_credential( - deposit_data.withdrawal_credentials, - spec - )); - dbg!(validator.has_eth1_withdrawal_credential(spec)); - dbg!(is_valid_deposit_signature(&deposit_data, spec).is_ok()); if is_compounding_withdrawal_credential(deposit_data.withdrawal_credentials, spec) && validator.has_eth1_withdrawal_credential(spec) && is_valid_deposit_signature(&deposit_data, spec).is_ok() @@ -500,8 +491,6 @@ pub fn apply_deposit( amount, ) }; - dbg!(effective_balance, state_balance); - dbg!(&deposit_data.pubkey, &deposit_data.withdrawal_credentials); // Create a new validator. let validator = Validator { pubkey: deposit_data.pubkey, @@ -529,7 +518,6 @@ pub fn apply_deposit( // [New in Electra:EIP7251] if let Ok(pending_balance_deposits) = state.pending_balance_deposits_mut() { - dbg!(new_validator_index, amount); pending_balance_deposits.push(PendingBalanceDeposit { index: new_validator_index as u64, amount, @@ -653,7 +641,7 @@ pub fn process_deposit_receipts( amount: receipt.amount, signature: receipt.signature.clone().into(), }; - apply_deposit(state, deposit_data, None, false, spec)? + apply_deposit(state, deposit_data, None, spec)? } Ok(()) diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 9a1fe609889..6b4bfd29cb5 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -2151,7 +2151,6 @@ impl BeaconState { if *balance > spec.min_activation_balance { let excess_balance = balance.safe_sub(spec.min_activation_balance)?; *balance = spec.min_activation_balance; - dbg!(validator_index, excess_balance); self.pending_balance_deposits_mut()? .push(PendingBalanceDeposit { index: validator_index as u64, @@ -2201,7 +2200,6 @@ impl BeaconState { if validator.has_eth1_withdrawal_credential(spec) { validator.withdrawal_credentials.as_fixed_bytes_mut()[0] = spec.compounding_withdrawal_prefix_byte; - dbg!(validator.withdrawal_credentials); self.queue_excess_active_balance(validator_index, spec)?; } Ok(()) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 3f862903615..f7adcbd75ee 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -171,9 +171,7 @@ impl Operation for Deposit { spec: &ChainSpec, _: &Operations, ) -> Result<(), BlockProcessingError> { - let res = process_deposits(state, &[self.clone()], spec); - dbg!(serde_json::to_string(state).unwrap()); - res + process_deposits(state, &[self.clone()], spec) } } diff --git a/testing/ef_tests/src/cases/sanity_blocks.rs b/testing/ef_tests/src/cases/sanity_blocks.rs index 3232ddee04e..91bb995cc43 100644 --- a/testing/ef_tests/src/cases/sanity_blocks.rs +++ b/testing/ef_tests/src/cases/sanity_blocks.rs @@ -60,9 +60,6 @@ impl Case for SanityBlocks { } fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> { - if _case_index != 74 || fork_name != ForkName::Electra { - return Ok(()); - } self.metadata.bls_setting.unwrap_or_default().check()?; let mut bulk_state = self.pre.clone(); @@ -114,15 +111,13 @@ impl Case for SanityBlocks { spec, )?; - // if block.state_root() == bulk_state.update_tree_hash_cache().unwrap() - // && block.state_root() == indiv_state.update_tree_hash_cache().unwrap() - // { - // Ok(()) - // } else { - // Err(BlockProcessingError::StateRootMismatch) - // } - - Ok::<_, BlockProcessingError>(()) + if block.state_root() == bulk_state.update_tree_hash_cache().unwrap() + && block.state_root() == indiv_state.update_tree_hash_cache().unwrap() + { + Ok(()) + } else { + Err(BlockProcessingError::StateRootMismatch) + } }) .map(|_| (bulk_state, indiv_state)); From a97e86c1a6fb15c9a67a94c5634fc9b9a341e14f Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 05:07:59 -0400 Subject: [PATCH 061/227] only increment the state deposit index on old deposit flow --- beacon_node/genesis/src/eth1_genesis_service.rs | 2 +- consensus/state_processing/src/genesis.rs | 2 +- .../src/per_block_processing/process_operations.rs | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/beacon_node/genesis/src/eth1_genesis_service.rs b/beacon_node/genesis/src/eth1_genesis_service.rs index 3e91504951c..70157050278 100644 --- a/beacon_node/genesis/src/eth1_genesis_service.rs +++ b/beacon_node/genesis/src/eth1_genesis_service.rs @@ -439,7 +439,7 @@ impl Eth1GenesisService { None }; - apply_deposit(&mut state, data, proof, spec) + apply_deposit(&mut state, data, proof, true, spec) .map_err(|e| format!("Error whilst processing deposit: {:?}", e)) })?; diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 38006b83cac..c73417077ae 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -38,7 +38,7 @@ pub fn initialize_beacon_state_from_eth1( .map_err(BlockProcessingError::MerkleTreeError)?; state.eth1_data_mut().deposit_root = deposit_tree.root(); let Deposit { proof, data } = deposit; - apply_deposit(&mut state, data, Some(proof), spec)?; + apply_deposit(&mut state, data, Some(proof), true, spec)?; } process_activations(&mut state, spec)?; diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 0fbb82ed1d9..9d2235e2d43 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -417,7 +417,7 @@ pub fn process_deposits( // Update the state in series. for deposit in deposits { - apply_deposit(state, deposit.data.clone(), None, spec)?; + apply_deposit(state, deposit.data.clone(), None, true, spec)?; } Ok(()) @@ -428,6 +428,7 @@ pub fn apply_deposit( state: &mut BeaconState, deposit_data: DepositData, proof: Option>, + increment_eth1_deposit_index: bool, spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { let deposit_index = state.eth1_deposit_index() as usize; @@ -440,7 +441,9 @@ pub fn apply_deposit( .map_err(|e| e.into_with_index(deposit_index))?; } - state.eth1_deposit_index_mut().safe_add_assign(1)?; + if increment_eth1_deposit_index { + state.eth1_deposit_index_mut().safe_add_assign(1)?; + } // Get an `Option` where `u64` is the validator index if this deposit public key // already exists in the beacon_state. @@ -641,7 +644,7 @@ pub fn process_deposit_receipts( amount: receipt.amount, signature: receipt.signature.clone().into(), }; - apply_deposit(state, deposit_data, None, spec)? + apply_deposit(state, deposit_data, None, false, spec)? } Ok(()) From 1e011da0e3813c4959843fb5f0c51c3efe650317 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 09:48:18 -0400 Subject: [PATCH 062/227] read electra payloads from file --- testing/ef_tests/src/cases/operations.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index f7adcbd75ee..730136e19c2 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -21,9 +21,9 @@ use state_processing::{ use std::fmt::Debug; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockBodyBellatrix, - BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconState, BlindedPayload, Deposit, - ExecutionPayload, FullPayload, ProposerSlashing, SignedBlsToExecutionChange, - SignedVoluntaryExit, SyncAggregate, + BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconState, + BlindedPayload, Deposit, ExecutionPayload, FullPayload, ProposerSlashing, + SignedBlsToExecutionChange, SignedVoluntaryExit, SyncAggregate, }; #[derive(Debug, Clone, Default, Deserialize)] @@ -299,6 +299,7 @@ impl Operation for BeaconBlockBody> { ForkName::Bellatrix => BeaconBlockBody::Bellatrix(<_>::from_ssz_bytes(bytes)?), ForkName::Capella => BeaconBlockBody::Capella(<_>::from_ssz_bytes(bytes)?), ForkName::Deneb => BeaconBlockBody::Deneb(<_>::from_ssz_bytes(bytes)?), + ForkName::Electra => BeaconBlockBody::Electra(<_>::from_ssz_bytes(bytes)?), _ => panic!(), }) }) @@ -350,6 +351,10 @@ impl Operation for BeaconBlockBody> { let inner = >>::from_ssz_bytes(bytes)?; BeaconBlockBody::Deneb(inner.clone_as_blinded()) } + ForkName::Electra => { + let inner = >>::from_ssz_bytes(bytes)?; + BeaconBlockBody::Electra(inner.clone_as_blinded()) + } _ => panic!(), }) }) From a7a8d2b8b26b57eeebde207a3c0e63b55418ee7e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 09:49:27 -0400 Subject: [PATCH 063/227] accept new payload v4 in mock el --- beacon_node/execution_layer/src/test_utils/handle_rpc.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index b9d9d76186a..9f1f244ef9a 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -98,7 +98,10 @@ pub async fn handle_rpc( .unwrap()) } } - ENGINE_NEW_PAYLOAD_V1 | ENGINE_NEW_PAYLOAD_V2 | ENGINE_NEW_PAYLOAD_V3 => { + ENGINE_NEW_PAYLOAD_V1 + | ENGINE_NEW_PAYLOAD_V2 + | ENGINE_NEW_PAYLOAD_V3 + | ENGINE_NEW_PAYLOAD_V4 => { let request = match method { ENGINE_NEW_PAYLOAD_V1 => JsonExecutionPayload::V1( get_param::>(params, 0) From 4041cb99affd13d9a429a0291f9768f45946a996 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 09:50:51 -0400 Subject: [PATCH 064/227] use correct max eb in epoch cache initialization --- consensus/state_processing/src/epoch_cache.rs | 7 ++++--- consensus/types/src/chain_spec.rs | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/consensus/state_processing/src/epoch_cache.rs b/consensus/state_processing/src/epoch_cache.rs index 3741f49c572..0e940fabe4f 100644 --- a/consensus/state_processing/src/epoch_cache.rs +++ b/consensus/state_processing/src/epoch_cache.rs @@ -86,9 +86,10 @@ impl PreEpochCache { let base_reward_per_increment = BaseRewardPerIncrement::new(total_active_balance, spec)?; let effective_balance_increment = spec.effective_balance_increment; - let max_effective_balance_eth = spec - .max_effective_balance - .safe_div(effective_balance_increment)?; + let max_effective_balance = + spec.max_effective_balance_for_fork(spec.fork_name_at_epoch(epoch)); + let max_effective_balance_eth = + max_effective_balance.safe_div(effective_balance_increment)?; let mut base_rewards = Vec::with_capacity(max_effective_balance_eth.safe_add(1)? as usize); diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 1cda156d8c2..81b2fecf1ed 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -400,6 +400,14 @@ impl ChainSpec { } } + pub fn max_effective_balance_for_fork(&self, fork_name: ForkName) -> u64 { + if fork_name >= ForkName::Electra { + self.max_effective_balance_electra + } else { + self.max_effective_balance + } + } + /// Returns a full `Fork` struct for a given epoch. pub fn fork_at_epoch(&self, epoch: Epoch) -> Fork { let current_fork_name = self.fork_name_at_epoch(epoch); From 740494f22f50062b1b1e342f56768e3ff20e435e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 09:58:14 -0400 Subject: [PATCH 065/227] drop initiate validator ordering optimization --- .../src/common/initiate_validator_exit.rs | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/consensus/state_processing/src/common/initiate_validator_exit.rs b/consensus/state_processing/src/common/initiate_validator_exit.rs index f917f66d428..d2d4374ab24 100644 --- a/consensus/state_processing/src/common/initiate_validator_exit.rs +++ b/consensus/state_processing/src/common/initiate_validator_exit.rs @@ -8,12 +8,12 @@ pub fn initiate_validator_exit( index: usize, spec: &ChainSpec, ) -> Result<(), Error> { - // We do things in a slightly different order to the spec here. Instead of immediately checking - // whether the validator has already exited, we instead prepare the exit cache and compute the - // cheap-to-calculate values from that. *Then* we look up the validator a single time in the - // validator tree (expensive), make the check and mutate as appropriate. Compared to the spec - // ordering, this saves us from looking up the validator in the validator registry multiple - // times. + let validator = state.get_validator_cow(index)?; + + // Return if the validator already initiated exit + if validator.exit_epoch != spec.far_future_epoch { + return Ok(()); + } // Ensure the exit cache is built. state.build_exit_cache(spec)?; @@ -36,14 +36,7 @@ pub fn initiate_validator_exit( exit_queue_epoch }; - let validator = state.get_validator_cow(index)?; - - // Return if the validator already initiated exit - if validator.exit_epoch != spec.far_future_epoch { - return Ok(()); - } - - let validator = validator.into_mut()?; + let validator = state.get_validator_mut(index)?; validator.exit_epoch = exit_queue_epoch; validator.withdrawable_epoch = exit_queue_epoch.safe_add(spec.min_validator_withdrawability_delay)?; From 25b1992d2acc18c9591f84533c0cd771147266b9 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 10:22:00 -0400 Subject: [PATCH 066/227] fix initiate exit for single pass --- .../src/per_epoch_processing/errors.rs | 2 + .../src/per_epoch_processing/single_pass.rs | 133 ++++++++++++++++-- 2 files changed, 121 insertions(+), 14 deletions(-) diff --git a/consensus/state_processing/src/per_epoch_processing/errors.rs b/consensus/state_processing/src/per_epoch_processing/errors.rs index 7f60f4a6e01..b6c9dbea521 100644 --- a/consensus/state_processing/src/per_epoch_processing/errors.rs +++ b/consensus/state_processing/src/per_epoch_processing/errors.rs @@ -26,6 +26,8 @@ pub enum EpochProcessingError { MilhouseError(milhouse::Error), EpochCache(EpochCacheError), SinglePassMissingActivationQueue, + MissingEarliestExitEpoch, + MissingExitBalanceToConsume, } impl From for EpochProcessingError { diff --git a/consensus/state_processing/src/per_epoch_processing/single_pass.rs b/consensus/state_processing/src/per_epoch_processing/single_pass.rs index 3d72f0cdd86..f5d3f974a6e 100644 --- a/consensus/state_processing/src/per_epoch_processing/single_pass.rs +++ b/consensus/state_processing/src/per_epoch_processing/single_pass.rs @@ -167,6 +167,9 @@ pub fn process_epoch_single_pass( None }; + let mut earliest_exit_epoch = state.earliest_exit_epoch().ok(); + let mut exit_balance_to_consume = state.exit_balance_to_consume().ok(); + // Split the state into several disjoint mutable borrows. let ( validators, @@ -286,6 +289,8 @@ pub fn process_epoch_single_pass( exit_cache, activation_queue_refs, state_ctxt, + earliest_exit_epoch.as_mut(), + exit_balance_to_consume.as_mut(), spec, )?; } @@ -320,6 +325,17 @@ pub fn process_epoch_single_pass( } } + if conf.registry_updates && fork_name >= ForkName::Electra { + if let Ok(earliest_exit_epoch_state) = state.earliest_exit_epoch_mut() { + *earliest_exit_epoch_state = + earliest_exit_epoch.ok_or(Error::MissingEarliestExitEpoch)?; + } + if let Ok(exit_balance_to_consume_state) = state.exit_balance_to_consume_mut() { + *exit_balance_to_consume_state = + exit_balance_to_consume.ok_or(Error::MissingExitBalanceToConsume)?; + } + } + // Finish processing pending balance deposits if relevant. // // This *could* be reordered after `process_pending_consolidations` which pushes only to the end @@ -527,12 +543,15 @@ impl RewardsAndPenaltiesContext { } } +#[allow(clippy::too_many_arguments)] fn process_single_registry_update( validator: &mut Cow, validator_info: &ValidatorInfo, exit_cache: &mut ExitCache, activation_queues: Option<(&BTreeSet, &mut ActivationQueue)>, state_ctxt: &StateContext, + earliest_exit_epoch: Option<&mut Epoch>, + exit_balance_to_consume: Option<&mut u64>, spec: &ChainSpec, ) -> Result<(), Error> { if state_ctxt.fork_name < ForkName::Electra { @@ -548,7 +567,14 @@ fn process_single_registry_update( spec, ) } else { - process_single_registry_update_post_electra(validator, exit_cache, state_ctxt, spec) + process_single_registry_update_post_electra( + validator, + exit_cache, + state_ctxt, + earliest_exit_epoch.ok_or(Error::MissingEarliestExitEpoch)?, + exit_balance_to_consume.ok_or(Error::MissingExitBalanceToConsume)?, + spec, + ) } } @@ -569,7 +595,7 @@ fn process_single_registry_update_pre_electra( if validator.is_active_at(current_epoch) && validator.effective_balance <= spec.ejection_balance { - initiate_validator_exit(validator, exit_cache, state_ctxt, spec)?; + initiate_validator_exit(validator, exit_cache, state_ctxt, None, None, spec)?; } if activation_queue.contains(&validator_info.index) { @@ -592,6 +618,8 @@ fn process_single_registry_update_post_electra( validator: &mut Cow, exit_cache: &mut ExitCache, state_ctxt: &StateContext, + earliest_exit_epoch: &mut Epoch, + exit_balance_to_consume: &mut u64, spec: &ChainSpec, ) -> Result<(), Error> { let current_epoch = state_ctxt.current_epoch; @@ -602,8 +630,14 @@ fn process_single_registry_update_post_electra( if validator.is_active_at(current_epoch) && validator.effective_balance <= spec.ejection_balance { - // TODO(electra): make sure initiate_validator_exit is updated - initiate_validator_exit(validator, exit_cache, state_ctxt, spec)?; + initiate_validator_exit( + validator, + exit_cache, + state_ctxt, + Some(earliest_exit_epoch), + Some(exit_balance_to_consume), + spec, + )?; } if validator.is_eligible_for_activation_with_finalized_checkpoint( @@ -621,6 +655,8 @@ fn initiate_validator_exit( validator: &mut Cow, exit_cache: &mut ExitCache, state_ctxt: &StateContext, + earliest_exit_epoch: Option<&mut Epoch>, + exit_balance_to_consume: Option<&mut u64>, spec: &ChainSpec, ) -> Result<(), Error> { // Return if the validator already initiated exit @@ -628,16 +664,27 @@ fn initiate_validator_exit( return Ok(()); } - // Compute exit queue epoch - let delayed_epoch = spec.compute_activation_exit_epoch(state_ctxt.current_epoch)?; - let mut exit_queue_epoch = exit_cache - .max_epoch()? - .map_or(delayed_epoch, |epoch| max(epoch, delayed_epoch)); - let exit_queue_churn = exit_cache.get_churn_at(exit_queue_epoch)?; - - if exit_queue_churn >= state_ctxt.churn_limit { - exit_queue_epoch.safe_add_assign(1)?; - } + let exit_queue_epoch = if state_ctxt.fork_name >= ForkName::Electra { + compute_exit_epoch_and_update_churn( + validator, + state_ctxt, + earliest_exit_epoch.ok_or(Error::MissingEarliestExitEpoch)?, + exit_balance_to_consume.ok_or(Error::MissingExitBalanceToConsume)?, + spec, + )? + } else { + // Compute exit queue epoch + let delayed_epoch = spec.compute_activation_exit_epoch(state_ctxt.current_epoch)?; + let mut exit_queue_epoch = exit_cache + .max_epoch()? + .map_or(delayed_epoch, |epoch| max(epoch, delayed_epoch)); + let exit_queue_churn = exit_cache.get_churn_at(exit_queue_epoch)?; + + if exit_queue_churn >= state_ctxt.churn_limit { + exit_queue_epoch.safe_add_assign(1)?; + } + exit_queue_epoch + }; let validator = validator.make_mut()?; validator.exit_epoch = exit_queue_epoch; @@ -648,6 +695,64 @@ fn initiate_validator_exit( Ok(()) } +fn compute_exit_epoch_and_update_churn( + validator: &mut Cow, + state_ctxt: &StateContext, + earliest_exit_epoch_state: &mut Epoch, + exit_balance_to_consume_state: &mut u64, + spec: &ChainSpec, +) -> Result { + let exit_balance = validator.effective_balance; + let mut earliest_exit_epoch = std::cmp::max( + *earliest_exit_epoch_state, + spec.compute_activation_exit_epoch(state_ctxt.current_epoch)?, + ); + + let per_epoch_churn = get_activation_exit_churn_limit(state_ctxt, spec)?; + // New epoch for exits + let mut exit_balance_to_consume = if *earliest_exit_epoch_state < earliest_exit_epoch { + per_epoch_churn + } else { + *exit_balance_to_consume_state + }; + + // Exit doesn't fit in the current earliest epoch + if exit_balance > exit_balance_to_consume { + let balance_to_process = exit_balance.safe_sub(exit_balance_to_consume)?; + let additional_epochs = balance_to_process + .safe_sub(1)? + .safe_div(per_epoch_churn)? + .safe_add(1)?; + earliest_exit_epoch.safe_add_assign(additional_epochs)?; + exit_balance_to_consume.safe_add_assign(additional_epochs.safe_mul(per_epoch_churn)?)?; + } + // Consume the balance and update state variables + *exit_balance_to_consume_state = exit_balance_to_consume.safe_sub(exit_balance)?; + *earliest_exit_epoch_state = earliest_exit_epoch; + + Ok(earliest_exit_epoch) +} + +fn get_activation_exit_churn_limit( + state_ctxt: &StateContext, + spec: &ChainSpec, +) -> Result { + Ok(std::cmp::min( + spec.max_per_epoch_activation_exit_churn_limit, + get_balance_churn_limit(state_ctxt, spec)?, + )) +} + +fn get_balance_churn_limit(state_ctxt: &StateContext, spec: &ChainSpec) -> Result { + let total_active_balance = state_ctxt.total_active_balance; + let churn = std::cmp::max( + spec.min_per_epoch_churn_limit_electra, + total_active_balance.safe_div(spec.churn_limit_quotient)?, + ); + + Ok(churn.safe_sub(churn.safe_rem(spec.effective_balance_increment)?)?) +} + impl SlashingsContext { fn new( state: &BeaconState, From a75257fb6e0063c1def98eea2b820e84f2a1fdb5 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 09:50:51 -0400 Subject: [PATCH 067/227] use correct max eb in epoch cache initialization --- consensus/state_processing/src/epoch_cache.rs | 7 ++++--- consensus/types/src/chain_spec.rs | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/consensus/state_processing/src/epoch_cache.rs b/consensus/state_processing/src/epoch_cache.rs index 3741f49c572..0e940fabe4f 100644 --- a/consensus/state_processing/src/epoch_cache.rs +++ b/consensus/state_processing/src/epoch_cache.rs @@ -86,9 +86,10 @@ impl PreEpochCache { let base_reward_per_increment = BaseRewardPerIncrement::new(total_active_balance, spec)?; let effective_balance_increment = spec.effective_balance_increment; - let max_effective_balance_eth = spec - .max_effective_balance - .safe_div(effective_balance_increment)?; + let max_effective_balance = + spec.max_effective_balance_for_fork(spec.fork_name_at_epoch(epoch)); + let max_effective_balance_eth = + max_effective_balance.safe_div(effective_balance_increment)?; let mut base_rewards = Vec::with_capacity(max_effective_balance_eth.safe_add(1)? as usize); diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 8ffafe6266d..df25713f023 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -400,6 +400,14 @@ impl ChainSpec { } } + pub fn max_effective_balance_for_fork(&self, fork_name: ForkName) -> u64 { + if fork_name >= ForkName::Electra { + self.max_effective_balance_electra + } else { + self.max_effective_balance + } + } + /// Returns a full `Fork` struct for a given epoch. pub fn fork_at_epoch(&self, epoch: Epoch) -> Fork { let current_fork_name = self.fork_name_at_epoch(epoch); From f4907ef971973b67fb692cb42a5119b5402a4371 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 09:58:14 -0400 Subject: [PATCH 068/227] drop initiate validator ordering optimization --- .../src/common/initiate_validator_exit.rs | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/consensus/state_processing/src/common/initiate_validator_exit.rs b/consensus/state_processing/src/common/initiate_validator_exit.rs index 8874e9ed4b8..49e3a7390d4 100644 --- a/consensus/state_processing/src/common/initiate_validator_exit.rs +++ b/consensus/state_processing/src/common/initiate_validator_exit.rs @@ -8,12 +8,12 @@ pub fn initiate_validator_exit( index: usize, spec: &ChainSpec, ) -> Result<(), Error> { - // We do things in a slightly different order to the spec here. Instead of immediately checking - // whether the validator has already exited, we instead prepare the exit cache and compute the - // cheap-to-calculate values from that. *Then* we look up the validator a single time in the - // validator tree (expensive), make the check and mutate as appropriate. Compared to the spec - // ordering, this saves us from looking up the validator in the validator registry multiple - // times. + let validator = state.get_validator_cow(index)?; + + // Return if the validator already initiated exit + if validator.exit_epoch != spec.far_future_epoch { + return Ok(()); + } // Ensure the exit cache is built. state.build_exit_cache(spec)?; @@ -36,14 +36,7 @@ pub fn initiate_validator_exit( exit_queue_epoch }; - let validator = state.get_validator_cow(index)?; - - // Return if the validator already initiated exit - if validator.exit_epoch != spec.far_future_epoch { - return Ok(()); - } - - let validator = validator.into_mut()?; + let validator = state.get_validator_mut(index)?; validator.exit_epoch = exit_queue_epoch; validator.withdrawable_epoch = exit_queue_epoch.safe_add(spec.min_validator_withdrawability_delay)?; From 75f22ee8443e4b485fd4813a1758c0710719af31 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 10:22:00 -0400 Subject: [PATCH 069/227] fix initiate exit for single pass --- .../src/per_epoch_processing/errors.rs | 2 + .../src/per_epoch_processing/single_pass.rs | 133 ++++++++++++++++-- 2 files changed, 121 insertions(+), 14 deletions(-) diff --git a/consensus/state_processing/src/per_epoch_processing/errors.rs b/consensus/state_processing/src/per_epoch_processing/errors.rs index 7f60f4a6e01..b6c9dbea521 100644 --- a/consensus/state_processing/src/per_epoch_processing/errors.rs +++ b/consensus/state_processing/src/per_epoch_processing/errors.rs @@ -26,6 +26,8 @@ pub enum EpochProcessingError { MilhouseError(milhouse::Error), EpochCache(EpochCacheError), SinglePassMissingActivationQueue, + MissingEarliestExitEpoch, + MissingExitBalanceToConsume, } impl From for EpochProcessingError { diff --git a/consensus/state_processing/src/per_epoch_processing/single_pass.rs b/consensus/state_processing/src/per_epoch_processing/single_pass.rs index 3d72f0cdd86..f5d3f974a6e 100644 --- a/consensus/state_processing/src/per_epoch_processing/single_pass.rs +++ b/consensus/state_processing/src/per_epoch_processing/single_pass.rs @@ -167,6 +167,9 @@ pub fn process_epoch_single_pass( None }; + let mut earliest_exit_epoch = state.earliest_exit_epoch().ok(); + let mut exit_balance_to_consume = state.exit_balance_to_consume().ok(); + // Split the state into several disjoint mutable borrows. let ( validators, @@ -286,6 +289,8 @@ pub fn process_epoch_single_pass( exit_cache, activation_queue_refs, state_ctxt, + earliest_exit_epoch.as_mut(), + exit_balance_to_consume.as_mut(), spec, )?; } @@ -320,6 +325,17 @@ pub fn process_epoch_single_pass( } } + if conf.registry_updates && fork_name >= ForkName::Electra { + if let Ok(earliest_exit_epoch_state) = state.earliest_exit_epoch_mut() { + *earliest_exit_epoch_state = + earliest_exit_epoch.ok_or(Error::MissingEarliestExitEpoch)?; + } + if let Ok(exit_balance_to_consume_state) = state.exit_balance_to_consume_mut() { + *exit_balance_to_consume_state = + exit_balance_to_consume.ok_or(Error::MissingExitBalanceToConsume)?; + } + } + // Finish processing pending balance deposits if relevant. // // This *could* be reordered after `process_pending_consolidations` which pushes only to the end @@ -527,12 +543,15 @@ impl RewardsAndPenaltiesContext { } } +#[allow(clippy::too_many_arguments)] fn process_single_registry_update( validator: &mut Cow, validator_info: &ValidatorInfo, exit_cache: &mut ExitCache, activation_queues: Option<(&BTreeSet, &mut ActivationQueue)>, state_ctxt: &StateContext, + earliest_exit_epoch: Option<&mut Epoch>, + exit_balance_to_consume: Option<&mut u64>, spec: &ChainSpec, ) -> Result<(), Error> { if state_ctxt.fork_name < ForkName::Electra { @@ -548,7 +567,14 @@ fn process_single_registry_update( spec, ) } else { - process_single_registry_update_post_electra(validator, exit_cache, state_ctxt, spec) + process_single_registry_update_post_electra( + validator, + exit_cache, + state_ctxt, + earliest_exit_epoch.ok_or(Error::MissingEarliestExitEpoch)?, + exit_balance_to_consume.ok_or(Error::MissingExitBalanceToConsume)?, + spec, + ) } } @@ -569,7 +595,7 @@ fn process_single_registry_update_pre_electra( if validator.is_active_at(current_epoch) && validator.effective_balance <= spec.ejection_balance { - initiate_validator_exit(validator, exit_cache, state_ctxt, spec)?; + initiate_validator_exit(validator, exit_cache, state_ctxt, None, None, spec)?; } if activation_queue.contains(&validator_info.index) { @@ -592,6 +618,8 @@ fn process_single_registry_update_post_electra( validator: &mut Cow, exit_cache: &mut ExitCache, state_ctxt: &StateContext, + earliest_exit_epoch: &mut Epoch, + exit_balance_to_consume: &mut u64, spec: &ChainSpec, ) -> Result<(), Error> { let current_epoch = state_ctxt.current_epoch; @@ -602,8 +630,14 @@ fn process_single_registry_update_post_electra( if validator.is_active_at(current_epoch) && validator.effective_balance <= spec.ejection_balance { - // TODO(electra): make sure initiate_validator_exit is updated - initiate_validator_exit(validator, exit_cache, state_ctxt, spec)?; + initiate_validator_exit( + validator, + exit_cache, + state_ctxt, + Some(earliest_exit_epoch), + Some(exit_balance_to_consume), + spec, + )?; } if validator.is_eligible_for_activation_with_finalized_checkpoint( @@ -621,6 +655,8 @@ fn initiate_validator_exit( validator: &mut Cow, exit_cache: &mut ExitCache, state_ctxt: &StateContext, + earliest_exit_epoch: Option<&mut Epoch>, + exit_balance_to_consume: Option<&mut u64>, spec: &ChainSpec, ) -> Result<(), Error> { // Return if the validator already initiated exit @@ -628,16 +664,27 @@ fn initiate_validator_exit( return Ok(()); } - // Compute exit queue epoch - let delayed_epoch = spec.compute_activation_exit_epoch(state_ctxt.current_epoch)?; - let mut exit_queue_epoch = exit_cache - .max_epoch()? - .map_or(delayed_epoch, |epoch| max(epoch, delayed_epoch)); - let exit_queue_churn = exit_cache.get_churn_at(exit_queue_epoch)?; - - if exit_queue_churn >= state_ctxt.churn_limit { - exit_queue_epoch.safe_add_assign(1)?; - } + let exit_queue_epoch = if state_ctxt.fork_name >= ForkName::Electra { + compute_exit_epoch_and_update_churn( + validator, + state_ctxt, + earliest_exit_epoch.ok_or(Error::MissingEarliestExitEpoch)?, + exit_balance_to_consume.ok_or(Error::MissingExitBalanceToConsume)?, + spec, + )? + } else { + // Compute exit queue epoch + let delayed_epoch = spec.compute_activation_exit_epoch(state_ctxt.current_epoch)?; + let mut exit_queue_epoch = exit_cache + .max_epoch()? + .map_or(delayed_epoch, |epoch| max(epoch, delayed_epoch)); + let exit_queue_churn = exit_cache.get_churn_at(exit_queue_epoch)?; + + if exit_queue_churn >= state_ctxt.churn_limit { + exit_queue_epoch.safe_add_assign(1)?; + } + exit_queue_epoch + }; let validator = validator.make_mut()?; validator.exit_epoch = exit_queue_epoch; @@ -648,6 +695,64 @@ fn initiate_validator_exit( Ok(()) } +fn compute_exit_epoch_and_update_churn( + validator: &mut Cow, + state_ctxt: &StateContext, + earliest_exit_epoch_state: &mut Epoch, + exit_balance_to_consume_state: &mut u64, + spec: &ChainSpec, +) -> Result { + let exit_balance = validator.effective_balance; + let mut earliest_exit_epoch = std::cmp::max( + *earliest_exit_epoch_state, + spec.compute_activation_exit_epoch(state_ctxt.current_epoch)?, + ); + + let per_epoch_churn = get_activation_exit_churn_limit(state_ctxt, spec)?; + // New epoch for exits + let mut exit_balance_to_consume = if *earliest_exit_epoch_state < earliest_exit_epoch { + per_epoch_churn + } else { + *exit_balance_to_consume_state + }; + + // Exit doesn't fit in the current earliest epoch + if exit_balance > exit_balance_to_consume { + let balance_to_process = exit_balance.safe_sub(exit_balance_to_consume)?; + let additional_epochs = balance_to_process + .safe_sub(1)? + .safe_div(per_epoch_churn)? + .safe_add(1)?; + earliest_exit_epoch.safe_add_assign(additional_epochs)?; + exit_balance_to_consume.safe_add_assign(additional_epochs.safe_mul(per_epoch_churn)?)?; + } + // Consume the balance and update state variables + *exit_balance_to_consume_state = exit_balance_to_consume.safe_sub(exit_balance)?; + *earliest_exit_epoch_state = earliest_exit_epoch; + + Ok(earliest_exit_epoch) +} + +fn get_activation_exit_churn_limit( + state_ctxt: &StateContext, + spec: &ChainSpec, +) -> Result { + Ok(std::cmp::min( + spec.max_per_epoch_activation_exit_churn_limit, + get_balance_churn_limit(state_ctxt, spec)?, + )) +} + +fn get_balance_churn_limit(state_ctxt: &StateContext, spec: &ChainSpec) -> Result { + let total_active_balance = state_ctxt.total_active_balance; + let churn = std::cmp::max( + spec.min_per_epoch_churn_limit_electra, + total_active_balance.safe_div(spec.churn_limit_quotient)?, + ); + + Ok(churn.safe_sub(churn.safe_rem(spec.effective_balance_increment)?)?) +} + impl SlashingsContext { fn new( state: &BeaconState, From fc2c942de4197955c67cbe1ccf5ccdd3d2e7ec38 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sat, 11 May 2024 09:49:27 -0400 Subject: [PATCH 070/227] accept new payload v4 in mock el --- beacon_node/execution_layer/src/test_utils/handle_rpc.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index b9d9d76186a..9f1f244ef9a 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -98,7 +98,10 @@ pub async fn handle_rpc( .unwrap()) } } - ENGINE_NEW_PAYLOAD_V1 | ENGINE_NEW_PAYLOAD_V2 | ENGINE_NEW_PAYLOAD_V3 => { + ENGINE_NEW_PAYLOAD_V1 + | ENGINE_NEW_PAYLOAD_V2 + | ENGINE_NEW_PAYLOAD_V3 + | ENGINE_NEW_PAYLOAD_V4 => { let request = match method { ENGINE_NEW_PAYLOAD_V1 => JsonExecutionPayload::V1( get_param::>(params, 0) From d7cc47ccc3582e4f5605269067f3e5b62292ceac Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 04:53:12 -0400 Subject: [PATCH 071/227] fix genesis from eth1 for electra --- consensus/state_processing/src/genesis.rs | 5 +- .../state_processing/src/upgrade/electra.rs | 97 +++++++++++-------- 2 files changed, 60 insertions(+), 42 deletions(-) diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index c73417077ae..049599ea945 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -2,9 +2,9 @@ use super::per_block_processing::{ errors::BlockProcessingError, process_operations::apply_deposit, }; use crate::common::DepositDataTree; +use crate::upgrade::electra::upgrade_state_to_electra; use crate::upgrade::{ upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, - upgrade_to_electra, }; use safe_arith::{ArithError, SafeArith}; use tree_hash::TreeHash; @@ -116,7 +116,8 @@ pub fn initialize_beacon_state_from_eth1( .electra_fork_epoch .map_or(false, |fork_epoch| fork_epoch == E::genesis_epoch()) { - upgrade_to_electra(&mut state, spec)?; + let post = upgrade_state_to_electra(&mut state, Epoch::new(0), Epoch::new(0), spec)?; + state = post; // Remove intermediate Deneb fork from `state.fork`. state.fork_mut().previous_version = spec.electra_fork_version; diff --git a/consensus/state_processing/src/upgrade/electra.rs b/consensus/state_processing/src/upgrade/electra.rs index 1e60bf488db..8d83e738282 100644 --- a/consensus/state_processing/src/upgrade/electra.rs +++ b/consensus/state_processing/src/upgrade/electra.rs @@ -1,8 +1,8 @@ use safe_arith::SafeArith; use std::mem; use types::{ - BeaconState, BeaconStateElectra, BeaconStateError as Error, ChainSpec, EpochCache, EthSpec, - Fork, + BeaconState, BeaconStateElectra, BeaconStateError as Error, ChainSpec, Epoch, EpochCache, + EthSpec, Fork, }; /// Transform a `Deneb` state into an `Electra` state. @@ -26,13 +26,66 @@ pub fn upgrade_to_electra( pre_state.build_total_active_balance_cache(spec)?; let earliest_consolidation_epoch = spec.compute_activation_exit_epoch(epoch)?; + let mut post = upgrade_state_to_electra( + pre_state, + earliest_exit_epoch, + earliest_consolidation_epoch, + spec, + )?; + + *post.exit_balance_to_consume_mut()? = post.get_activation_exit_churn_limit(spec)?; + *post.consolidation_balance_to_consume_mut()? = post.get_consolidation_churn_limit(spec)?; + + // Add validators that are not yet active to pending balance deposits + let validators = post.validators().clone(); + let mut pre_activation = validators + .iter() + .enumerate() + .filter(|(_, validator)| validator.activation_epoch == spec.far_future_epoch) + .collect::>(); + + // Sort the indices by activation_eligibility_epoch and then by index + pre_activation.sort_by(|(index_a, val_a), (index_b, val_b)| { + if val_a.activation_eligibility_epoch == val_b.activation_eligibility_epoch { + index_a.cmp(index_b) + } else { + val_a + .activation_eligibility_epoch + .cmp(&val_b.activation_eligibility_epoch) + } + }); + + // Process validators to queue entire balance and reset them + for (index, _) in pre_activation { + post.queue_entire_balance_and_reset_validator(index, spec)?; + } + + // Ensure early adopters of compounding credentials go through the activation churn + for (index, validator) in validators.iter().enumerate() { + if validator.has_compounding_withdrawal_credential(spec) { + post.queue_excess_active_balance(index, spec)?; + } + } + + *pre_state = post; + + Ok(()) +} + +pub fn upgrade_state_to_electra( + pre_state: &mut BeaconState, + earliest_exit_epoch: Epoch, + earliest_consolidation_epoch: Epoch, + spec: &ChainSpec, +) -> Result, Error> { + let epoch = pre_state.current_epoch(); let pre = pre_state.as_deneb_mut()?; // Where possible, use something like `mem::take` to move fields from behind the &mut // reference. For other fields that don't have a good default value, use `clone`. // // Fixed size vectors get cloned because replacing them would require the same size // allocation as cloning. - let mut post = BeaconState::Electra(BeaconStateElectra { + let post = BeaconState::Electra(BeaconStateElectra { // Versioning genesis_time: pre.genesis_time, genesis_validators_root: pre.genesis_validators_root, @@ -96,41 +149,5 @@ pub fn upgrade_to_electra( slashings_cache: mem::take(&mut pre.slashings_cache), epoch_cache: EpochCache::default(), }); - *post.exit_balance_to_consume_mut()? = post.get_activation_exit_churn_limit(spec)?; - *post.consolidation_balance_to_consume_mut()? = post.get_consolidation_churn_limit(spec)?; - - // Add validators that are not yet active to pending balance deposits - let validators = post.validators().clone(); - let mut pre_activation = validators - .iter() - .enumerate() - .filter(|(_, validator)| validator.activation_epoch == spec.far_future_epoch) - .collect::>(); - - // Sort the indices by activation_eligibility_epoch and then by index - pre_activation.sort_by(|(index_a, val_a), (index_b, val_b)| { - if val_a.activation_eligibility_epoch == val_b.activation_eligibility_epoch { - index_a.cmp(index_b) - } else { - val_a - .activation_eligibility_epoch - .cmp(&val_b.activation_eligibility_epoch) - } - }); - - // Process validators to queue entire balance and reset them - for (index, _) in pre_activation { - post.queue_entire_balance_and_reset_validator(index, spec)?; - } - - // Ensure early adopters of compounding credentials go through the activation churn - for (index, validator) in validators.iter().enumerate() { - if validator.has_compounding_withdrawal_credential(spec) { - post.queue_excess_active_balance(index, spec)?; - } - } - - *pre_state = post; - - Ok(()) + Ok(post) } From 3b1fb0ad81451413e5d86731eefcda1af72e2d08 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Sun, 12 May 2024 04:24:19 -0500 Subject: [PATCH 072/227] Fix Electra Fork Choice Tests (#5764) --- beacon_node/store/src/consensus_context.rs | 5 +- .../state_processing/src/consensus_context.rs | 61 ++++++------------- 2 files changed, 22 insertions(+), 44 deletions(-) diff --git a/beacon_node/store/src/consensus_context.rs b/beacon_node/store/src/consensus_context.rs index 409b364992d..281106d9aaa 100644 --- a/beacon_node/store/src/consensus_context.rs +++ b/beacon_node/store/src/consensus_context.rs @@ -1,7 +1,7 @@ use ssz_derive::{Decode, Encode}; use state_processing::ConsensusContext; use std::collections::HashMap; -use types::{AttestationData, BitList, EthSpec, Hash256, IndexedAttestation, Slot}; +use types::{EthSpec, Hash256, IndexedAttestation, Slot}; /// The consensus context is stored on disk as part of the data availability overflow cache. /// @@ -21,8 +21,7 @@ pub struct OnDiskConsensusContext { /// /// They are not part of the on-disk format. #[ssz(skip_serializing, skip_deserializing)] - indexed_attestations: - HashMap<(AttestationData, BitList), IndexedAttestation>, + indexed_attestations: HashMap>, } impl OnDiskConsensusContext { diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index 6b0f2eb8fee..b0eaf3422d3 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -4,9 +4,8 @@ use crate::EpochCacheError; use std::collections::{hash_map::Entry, HashMap}; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, AttestationData, AttestationRef, BeaconState, BeaconStateError, BitList, - ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, IndexedAttestationRef, - SignedBeaconBlock, Slot, + AbstractExecPayload, AttestationRef, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, + Hash256, IndexedAttestation, IndexedAttestationRef, SignedBeaconBlock, Slot, }; #[derive(Debug, PartialEq, Clone)] @@ -22,8 +21,7 @@ pub struct ConsensusContext { /// Block root of the block at `slot`. pub current_block_root: Option, /// Cache of indexed attestations constructed during block processing. - pub indexed_attestations: - HashMap<(AttestationData, BitList), IndexedAttestation>, + pub indexed_attestations: HashMap>, } #[derive(Debug, PartialEq, Clone)] @@ -154,41 +152,25 @@ impl ConsensusContext { state: &BeaconState, attestation: AttestationRef<'a, E>, ) -> Result, BlockOperationError> { + let key = attestation.tree_hash_root(); match attestation { - AttestationRef::Base(attn) => { - let extended_aggregation_bits = attn - .extend_aggregation_bits() - .map_err(BeaconStateError::from)?; - - let key = (attn.data.clone(), extended_aggregation_bits); - - match self.indexed_attestations.entry(key) { - Entry::Occupied(occupied) => Ok(occupied.into_mut()), - Entry::Vacant(vacant) => { - let committee = - state.get_beacon_committee(attn.data.slot, attn.data.index)?; - let indexed_attestation = attesting_indices_base::get_indexed_attestation( - committee.committee, - attn, - )?; - Ok(vacant.insert(indexed_attestation)) - } + AttestationRef::Base(attn) => match self.indexed_attestations.entry(key) { + Entry::Occupied(occupied) => Ok(occupied.into_mut()), + Entry::Vacant(vacant) => { + let committee = state.get_beacon_committee(attn.data.slot, attn.data.index)?; + let indexed_attestation = + attesting_indices_base::get_indexed_attestation(committee.committee, attn)?; + Ok(vacant.insert(indexed_attestation)) } - } - AttestationRef::Electra(attn) => { - let key = (attn.data.clone(), attn.aggregation_bits.clone()); - - match self.indexed_attestations.entry(key) { - Entry::Occupied(occupied) => Ok(occupied.into_mut()), - Entry::Vacant(vacant) => { - let indexed_attestation = - attesting_indices_electra::get_indexed_attestation_from_state( - state, attn, - )?; - Ok(vacant.insert(indexed_attestation)) - } + }, + AttestationRef::Electra(attn) => match self.indexed_attestations.entry(key) { + Entry::Occupied(occupied) => Ok(occupied.into_mut()), + Entry::Vacant(vacant) => { + let indexed_attestation = + attesting_indices_electra::get_indexed_attestation_from_state(state, attn)?; + Ok(vacant.insert(indexed_attestation)) } - } + }, } .map(|indexed_attestation| (*indexed_attestation).to_ref()) } @@ -200,10 +182,7 @@ impl ConsensusContext { #[must_use] pub fn set_indexed_attestations( mut self, - attestations: HashMap< - (AttestationData, BitList), - IndexedAttestation, - >, + attestations: HashMap>, ) -> Self { self.indexed_attestations = attestations; self From 8d4a9216120635f6b093584082b10c3ee7bc59af Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 05:34:16 -0400 Subject: [PATCH 073/227] add new operation tests --- testing/ef_tests/src/cases/operations.rs | 77 +++++++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 730136e19c2..b5906ceee5e 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -6,6 +6,9 @@ use serde::Deserialize; use ssz::Decode; use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache; use state_processing::epoch_cache::initialize_epoch_cache; +use state_processing::per_block_processing::process_operations::{ + process_consolidations, process_deposit_receipts, process_execution_layer_withdrawal_requests, +}; use state_processing::{ per_block_processing::{ errors::BlockProcessingError, @@ -22,8 +25,9 @@ use std::fmt::Debug; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockBodyBellatrix, BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconState, - BlindedPayload, Deposit, ExecutionPayload, FullPayload, ProposerSlashing, - SignedBlsToExecutionChange, SignedVoluntaryExit, SyncAggregate, + BlindedPayload, Deposit, DepositReceipt, ExecutionLayerWithdrawalRequest, ExecutionPayload, + FullPayload, ProposerSlashing, SignedBlsToExecutionChange, SignedConsolidation, + SignedVoluntaryExit, SyncAggregate, }; #[derive(Debug, Clone, Default, Deserialize)] @@ -437,6 +441,75 @@ impl Operation for SignedBlsToExecutionChange { } } +impl Operation for ExecutionLayerWithdrawalRequest { + fn handler_name() -> String { + "execution_layer_withdrawal_request".into() + } + + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name >= ForkName::Electra + } + + fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { + ssz_decode_file(path) + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + _extra: &Operations, + ) -> Result<(), BlockProcessingError> { + process_execution_layer_withdrawal_requests(state, &[self.clone()], spec) + } +} + +impl Operation for DepositReceipt { + fn handler_name() -> String { + "deposit_receipt".into() + } + + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name >= ForkName::Electra + } + + fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { + ssz_decode_file(path) + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + _extra: &Operations, + ) -> Result<(), BlockProcessingError> { + process_deposit_receipts(state, &[self.clone()], spec) + } +} + +impl Operation for SignedConsolidation { + fn handler_name() -> String { + "consolidation".into() + } + + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name >= ForkName::Electra + } + + fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { + ssz_decode_file(path) + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + _extra: &Operations, + ) -> Result<(), BlockProcessingError> { + process_consolidations(state, &[self.clone()], VerifySignatures::True, spec) + } +} + impl> LoadCase for Operations { fn load_from_dir(path: &Path, fork_name: ForkName) -> Result { let spec = &testing_spec::(fork_name); From 6b7b7aaf7c3851b25d4a4234ec68a0a643730b27 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Sun, 12 May 2024 04:24:19 -0500 Subject: [PATCH 074/227] Fix Electra Fork Choice Tests (#5764) --- beacon_node/store/src/consensus_context.rs | 5 +- .../state_processing/src/consensus_context.rs | 61 ++++++------------- 2 files changed, 22 insertions(+), 44 deletions(-) diff --git a/beacon_node/store/src/consensus_context.rs b/beacon_node/store/src/consensus_context.rs index 409b364992d..281106d9aaa 100644 --- a/beacon_node/store/src/consensus_context.rs +++ b/beacon_node/store/src/consensus_context.rs @@ -1,7 +1,7 @@ use ssz_derive::{Decode, Encode}; use state_processing::ConsensusContext; use std::collections::HashMap; -use types::{AttestationData, BitList, EthSpec, Hash256, IndexedAttestation, Slot}; +use types::{EthSpec, Hash256, IndexedAttestation, Slot}; /// The consensus context is stored on disk as part of the data availability overflow cache. /// @@ -21,8 +21,7 @@ pub struct OnDiskConsensusContext { /// /// They are not part of the on-disk format. #[ssz(skip_serializing, skip_deserializing)] - indexed_attestations: - HashMap<(AttestationData, BitList), IndexedAttestation>, + indexed_attestations: HashMap>, } impl OnDiskConsensusContext { diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index 6b0f2eb8fee..b0eaf3422d3 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -4,9 +4,8 @@ use crate::EpochCacheError; use std::collections::{hash_map::Entry, HashMap}; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, AttestationData, AttestationRef, BeaconState, BeaconStateError, BitList, - ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, IndexedAttestationRef, - SignedBeaconBlock, Slot, + AbstractExecPayload, AttestationRef, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, + Hash256, IndexedAttestation, IndexedAttestationRef, SignedBeaconBlock, Slot, }; #[derive(Debug, PartialEq, Clone)] @@ -22,8 +21,7 @@ pub struct ConsensusContext { /// Block root of the block at `slot`. pub current_block_root: Option, /// Cache of indexed attestations constructed during block processing. - pub indexed_attestations: - HashMap<(AttestationData, BitList), IndexedAttestation>, + pub indexed_attestations: HashMap>, } #[derive(Debug, PartialEq, Clone)] @@ -154,41 +152,25 @@ impl ConsensusContext { state: &BeaconState, attestation: AttestationRef<'a, E>, ) -> Result, BlockOperationError> { + let key = attestation.tree_hash_root(); match attestation { - AttestationRef::Base(attn) => { - let extended_aggregation_bits = attn - .extend_aggregation_bits() - .map_err(BeaconStateError::from)?; - - let key = (attn.data.clone(), extended_aggregation_bits); - - match self.indexed_attestations.entry(key) { - Entry::Occupied(occupied) => Ok(occupied.into_mut()), - Entry::Vacant(vacant) => { - let committee = - state.get_beacon_committee(attn.data.slot, attn.data.index)?; - let indexed_attestation = attesting_indices_base::get_indexed_attestation( - committee.committee, - attn, - )?; - Ok(vacant.insert(indexed_attestation)) - } + AttestationRef::Base(attn) => match self.indexed_attestations.entry(key) { + Entry::Occupied(occupied) => Ok(occupied.into_mut()), + Entry::Vacant(vacant) => { + let committee = state.get_beacon_committee(attn.data.slot, attn.data.index)?; + let indexed_attestation = + attesting_indices_base::get_indexed_attestation(committee.committee, attn)?; + Ok(vacant.insert(indexed_attestation)) } - } - AttestationRef::Electra(attn) => { - let key = (attn.data.clone(), attn.aggregation_bits.clone()); - - match self.indexed_attestations.entry(key) { - Entry::Occupied(occupied) => Ok(occupied.into_mut()), - Entry::Vacant(vacant) => { - let indexed_attestation = - attesting_indices_electra::get_indexed_attestation_from_state( - state, attn, - )?; - Ok(vacant.insert(indexed_attestation)) - } + }, + AttestationRef::Electra(attn) => match self.indexed_attestations.entry(key) { + Entry::Occupied(occupied) => Ok(occupied.into_mut()), + Entry::Vacant(vacant) => { + let indexed_attestation = + attesting_indices_electra::get_indexed_attestation_from_state(state, attn)?; + Ok(vacant.insert(indexed_attestation)) } - } + }, } .map(|indexed_attestation| (*indexed_attestation).to_ref()) } @@ -200,10 +182,7 @@ impl ConsensusContext { #[must_use] pub fn set_indexed_attestations( mut self, - attestations: HashMap< - (AttestationData, BitList), - IndexedAttestation, - >, + attestations: HashMap>, ) -> Self { self.indexed_attestations = attestations; self From af7ba6ff70241589df3159e35e6fa36ce297ac50 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Sun, 12 May 2024 04:24:19 -0500 Subject: [PATCH 075/227] Fix Electra Fork Choice Tests (#5764) --- beacon_node/store/src/consensus_context.rs | 5 +- .../state_processing/src/consensus_context.rs | 61 ++++++------------- 2 files changed, 22 insertions(+), 44 deletions(-) diff --git a/beacon_node/store/src/consensus_context.rs b/beacon_node/store/src/consensus_context.rs index 409b364992d..281106d9aaa 100644 --- a/beacon_node/store/src/consensus_context.rs +++ b/beacon_node/store/src/consensus_context.rs @@ -1,7 +1,7 @@ use ssz_derive::{Decode, Encode}; use state_processing::ConsensusContext; use std::collections::HashMap; -use types::{AttestationData, BitList, EthSpec, Hash256, IndexedAttestation, Slot}; +use types::{EthSpec, Hash256, IndexedAttestation, Slot}; /// The consensus context is stored on disk as part of the data availability overflow cache. /// @@ -21,8 +21,7 @@ pub struct OnDiskConsensusContext { /// /// They are not part of the on-disk format. #[ssz(skip_serializing, skip_deserializing)] - indexed_attestations: - HashMap<(AttestationData, BitList), IndexedAttestation>, + indexed_attestations: HashMap>, } impl OnDiskConsensusContext { diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index 6b0f2eb8fee..b0eaf3422d3 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -4,9 +4,8 @@ use crate::EpochCacheError; use std::collections::{hash_map::Entry, HashMap}; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, AttestationData, AttestationRef, BeaconState, BeaconStateError, BitList, - ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, IndexedAttestationRef, - SignedBeaconBlock, Slot, + AbstractExecPayload, AttestationRef, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, + Hash256, IndexedAttestation, IndexedAttestationRef, SignedBeaconBlock, Slot, }; #[derive(Debug, PartialEq, Clone)] @@ -22,8 +21,7 @@ pub struct ConsensusContext { /// Block root of the block at `slot`. pub current_block_root: Option, /// Cache of indexed attestations constructed during block processing. - pub indexed_attestations: - HashMap<(AttestationData, BitList), IndexedAttestation>, + pub indexed_attestations: HashMap>, } #[derive(Debug, PartialEq, Clone)] @@ -154,41 +152,25 @@ impl ConsensusContext { state: &BeaconState, attestation: AttestationRef<'a, E>, ) -> Result, BlockOperationError> { + let key = attestation.tree_hash_root(); match attestation { - AttestationRef::Base(attn) => { - let extended_aggregation_bits = attn - .extend_aggregation_bits() - .map_err(BeaconStateError::from)?; - - let key = (attn.data.clone(), extended_aggregation_bits); - - match self.indexed_attestations.entry(key) { - Entry::Occupied(occupied) => Ok(occupied.into_mut()), - Entry::Vacant(vacant) => { - let committee = - state.get_beacon_committee(attn.data.slot, attn.data.index)?; - let indexed_attestation = attesting_indices_base::get_indexed_attestation( - committee.committee, - attn, - )?; - Ok(vacant.insert(indexed_attestation)) - } + AttestationRef::Base(attn) => match self.indexed_attestations.entry(key) { + Entry::Occupied(occupied) => Ok(occupied.into_mut()), + Entry::Vacant(vacant) => { + let committee = state.get_beacon_committee(attn.data.slot, attn.data.index)?; + let indexed_attestation = + attesting_indices_base::get_indexed_attestation(committee.committee, attn)?; + Ok(vacant.insert(indexed_attestation)) } - } - AttestationRef::Electra(attn) => { - let key = (attn.data.clone(), attn.aggregation_bits.clone()); - - match self.indexed_attestations.entry(key) { - Entry::Occupied(occupied) => Ok(occupied.into_mut()), - Entry::Vacant(vacant) => { - let indexed_attestation = - attesting_indices_electra::get_indexed_attestation_from_state( - state, attn, - )?; - Ok(vacant.insert(indexed_attestation)) - } + }, + AttestationRef::Electra(attn) => match self.indexed_attestations.entry(key) { + Entry::Occupied(occupied) => Ok(occupied.into_mut()), + Entry::Vacant(vacant) => { + let indexed_attestation = + attesting_indices_electra::get_indexed_attestation_from_state(state, attn)?; + Ok(vacant.insert(indexed_attestation)) } - } + }, } .map(|indexed_attestation| (*indexed_attestation).to_ref()) } @@ -200,10 +182,7 @@ impl ConsensusContext { #[must_use] pub fn set_indexed_attestations( mut self, - attestations: HashMap< - (AttestationData, BitList), - IndexedAttestation, - >, + attestations: HashMap>, ) -> Self { self.indexed_attestations = attestations; self From aaf8e503c67fa18eb6d9b2649b242293881b91f4 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Sun, 12 May 2024 04:24:19 -0500 Subject: [PATCH 076/227] Fix Electra Fork Choice Tests (#5764) --- beacon_node/store/src/consensus_context.rs | 5 +- .../state_processing/src/consensus_context.rs | 61 ++++++------------- 2 files changed, 22 insertions(+), 44 deletions(-) diff --git a/beacon_node/store/src/consensus_context.rs b/beacon_node/store/src/consensus_context.rs index 409b364992d..281106d9aaa 100644 --- a/beacon_node/store/src/consensus_context.rs +++ b/beacon_node/store/src/consensus_context.rs @@ -1,7 +1,7 @@ use ssz_derive::{Decode, Encode}; use state_processing::ConsensusContext; use std::collections::HashMap; -use types::{AttestationData, BitList, EthSpec, Hash256, IndexedAttestation, Slot}; +use types::{EthSpec, Hash256, IndexedAttestation, Slot}; /// The consensus context is stored on disk as part of the data availability overflow cache. /// @@ -21,8 +21,7 @@ pub struct OnDiskConsensusContext { /// /// They are not part of the on-disk format. #[ssz(skip_serializing, skip_deserializing)] - indexed_attestations: - HashMap<(AttestationData, BitList), IndexedAttestation>, + indexed_attestations: HashMap>, } impl OnDiskConsensusContext { diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index 6b0f2eb8fee..b0eaf3422d3 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -4,9 +4,8 @@ use crate::EpochCacheError; use std::collections::{hash_map::Entry, HashMap}; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, AttestationData, AttestationRef, BeaconState, BeaconStateError, BitList, - ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, IndexedAttestationRef, - SignedBeaconBlock, Slot, + AbstractExecPayload, AttestationRef, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, + Hash256, IndexedAttestation, IndexedAttestationRef, SignedBeaconBlock, Slot, }; #[derive(Debug, PartialEq, Clone)] @@ -22,8 +21,7 @@ pub struct ConsensusContext { /// Block root of the block at `slot`. pub current_block_root: Option, /// Cache of indexed attestations constructed during block processing. - pub indexed_attestations: - HashMap<(AttestationData, BitList), IndexedAttestation>, + pub indexed_attestations: HashMap>, } #[derive(Debug, PartialEq, Clone)] @@ -154,41 +152,25 @@ impl ConsensusContext { state: &BeaconState, attestation: AttestationRef<'a, E>, ) -> Result, BlockOperationError> { + let key = attestation.tree_hash_root(); match attestation { - AttestationRef::Base(attn) => { - let extended_aggregation_bits = attn - .extend_aggregation_bits() - .map_err(BeaconStateError::from)?; - - let key = (attn.data.clone(), extended_aggregation_bits); - - match self.indexed_attestations.entry(key) { - Entry::Occupied(occupied) => Ok(occupied.into_mut()), - Entry::Vacant(vacant) => { - let committee = - state.get_beacon_committee(attn.data.slot, attn.data.index)?; - let indexed_attestation = attesting_indices_base::get_indexed_attestation( - committee.committee, - attn, - )?; - Ok(vacant.insert(indexed_attestation)) - } + AttestationRef::Base(attn) => match self.indexed_attestations.entry(key) { + Entry::Occupied(occupied) => Ok(occupied.into_mut()), + Entry::Vacant(vacant) => { + let committee = state.get_beacon_committee(attn.data.slot, attn.data.index)?; + let indexed_attestation = + attesting_indices_base::get_indexed_attestation(committee.committee, attn)?; + Ok(vacant.insert(indexed_attestation)) } - } - AttestationRef::Electra(attn) => { - let key = (attn.data.clone(), attn.aggregation_bits.clone()); - - match self.indexed_attestations.entry(key) { - Entry::Occupied(occupied) => Ok(occupied.into_mut()), - Entry::Vacant(vacant) => { - let indexed_attestation = - attesting_indices_electra::get_indexed_attestation_from_state( - state, attn, - )?; - Ok(vacant.insert(indexed_attestation)) - } + }, + AttestationRef::Electra(attn) => match self.indexed_attestations.entry(key) { + Entry::Occupied(occupied) => Ok(occupied.into_mut()), + Entry::Vacant(vacant) => { + let indexed_attestation = + attesting_indices_electra::get_indexed_attestation_from_state(state, attn)?; + Ok(vacant.insert(indexed_attestation)) } - } + }, } .map(|indexed_attestation| (*indexed_attestation).to_ref()) } @@ -200,10 +182,7 @@ impl ConsensusContext { #[must_use] pub fn set_indexed_attestations( mut self, - attestations: HashMap< - (AttestationData, BitList), - IndexedAttestation, - >, + attestations: HashMap>, ) -> Self { self.indexed_attestations = attestations; self From 89ef043491461237e982bc86ad3717ab64b4e075 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 06:53:52 -0400 Subject: [PATCH 077/227] add epoch processing new tests --- .../ef_tests/src/cases/epoch_processing.rs | 37 +++++++++++++++++++ testing/ef_tests/tests/tests.rs | 12 ++++++ 2 files changed, 49 insertions(+) diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index c4c592e4cf2..5943e72b570 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -10,6 +10,9 @@ use state_processing::per_epoch_processing::capella::process_historical_summarie use state_processing::per_epoch_processing::effective_balance_updates::{ process_effective_balance_updates, process_effective_balance_updates_slow, }; +use state_processing::per_epoch_processing::single_pass::{ + process_epoch_single_pass, SinglePassConfig, +}; use state_processing::per_epoch_processing::{ altair, base, historical_roots_update::process_historical_roots_update, @@ -53,6 +56,10 @@ pub struct Slashings; #[derive(Debug)] pub struct Eth1DataReset; #[derive(Debug)] +pub struct PendingBalanceDeposits; +#[derive(Debug)] +pub struct PendingConsolidations; +#[derive(Debug)] pub struct EffectiveBalanceUpdates; #[derive(Debug)] pub struct SlashingsReset; @@ -79,6 +86,8 @@ type_name!(RewardsAndPenalties, "rewards_and_penalties"); type_name!(RegistryUpdates, "registry_updates"); type_name!(Slashings, "slashings"); type_name!(Eth1DataReset, "eth1_data_reset"); +type_name!(PendingBalanceDeposits, "pending_balance_deposits"); +type_name!(PendingConsolidations, "pending_consolidations"); type_name!(EffectiveBalanceUpdates, "effective_balance_updates"); type_name!(SlashingsReset, "slashings_reset"); type_name!(RandaoMixesReset, "randao_mixes_reset"); @@ -178,6 +187,34 @@ impl EpochTransition for Eth1DataReset { } } +impl EpochTransition for PendingBalanceDeposits { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_epoch_single_pass( + state, + spec, + SinglePassConfig { + pending_balance_deposits: true, + ..SinglePassConfig::disable_all() + }, + ) + .map(|_| ()) + } +} + +impl EpochTransition for PendingConsolidations { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_epoch_single_pass( + state, + spec, + SinglePassConfig { + pending_consolidations: true, + ..SinglePassConfig::disable_all() + }, + ) + .map(|_| ()) + } +} + impl EpochTransition for EffectiveBalanceUpdates { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { if let BeaconState::Base(_) = state { diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 1947dd4b419..f9879cdbcf9 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -664,6 +664,18 @@ fn epoch_processing_eth1_data_reset() { EpochProcessingHandler::::default().run(); } +#[test] +fn epoch_processing_pending_balance_deposits() { + EpochProcessingHandler::::default().run(); + EpochProcessingHandler::::default().run(); +} + +#[test] +fn epoch_processing_pending_consolidations() { + EpochProcessingHandler::::default().run(); + EpochProcessingHandler::::default().run(); +} + #[test] fn epoch_processing_effective_balance_updates() { EpochProcessingHandler::::default().run(); From e061e1859dba6cecb4dd965c8accc4dbe81de436 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 07:08:00 -0400 Subject: [PATCH 078/227] fix ef test compilation --- testing/ef_tests/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/ef_tests/src/lib.rs b/testing/ef_tests/src/lib.rs index 5ab2b4b7b43..e55551be701 100644 --- a/testing/ef_tests/src/lib.rs +++ b/testing/ef_tests/src/lib.rs @@ -3,8 +3,8 @@ pub use cases::WithdrawalsPayload; pub use cases::{ Case, EffectiveBalanceUpdates, Eth1DataReset, HistoricalRootsUpdate, HistoricalSummariesUpdate, InactivityUpdates, JustificationAndFinalization, ParticipationFlagUpdates, - ParticipationRecordUpdates, RandaoMixesReset, RegistryUpdates, RewardsAndPenalties, Slashings, - SlashingsReset, SyncCommitteeUpdates, + ParticipationRecordUpdates, PendingBalanceDeposits, PendingConsolidations, RandaoMixesReset, + RegistryUpdates, RewardsAndPenalties, Slashings, SlashingsReset, SyncCommitteeUpdates, }; pub use decode::log_file_access; pub use error::Error; From c37a4565f107f160b76393d82cab40243cc88540 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 07:37:31 -0400 Subject: [PATCH 079/227] disable and enable ef test for epoch processing appropriately --- .../ef_tests/src/cases/epoch_processing.rs | 57 +++++++++++++------ 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index 5943e72b570..302c5fa42e8 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -203,6 +203,7 @@ impl EpochTransition for PendingBalanceDeposits { impl EpochTransition for PendingConsolidations { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + initialize_epoch_cache(state, spec)?; process_epoch_single_pass( state, spec, @@ -341,24 +342,44 @@ impl> Case for EpochProcessing { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - match fork_name { - // No Altair tests for genesis fork. - ForkName::Base => { - T::name() != "sync_committee_updates" - && T::name() != "inactivity_updates" - && T::name() != "participation_flag_updates" - && T::name() != "historical_summaries_update" - } - // No phase0 tests for Altair and later. - ForkName::Altair | ForkName::Bellatrix => { - T::name() != "participation_record_updates" - && T::name() != "historical_summaries_update" - } - ForkName::Capella | ForkName::Deneb | ForkName::Electra => { - T::name() != "participation_record_updates" - && T::name() != "historical_roots_update" - } - } + // Deprecated tests + let deprecated_in_altair = if fork_name > ForkName::Base { + T::name() != "participation_record_updates" + && T::name() != "historical_summaries_update" + } else { + false + }; + let deprecated_in_bellatrix = if fork_name > ForkName::Altair { + T::name() != "historical_roots_update" + } else { + false + }; + let deprecated_in_capella = if fork_name > ForkName::Bellatrix { + T::name() != "historical_summaries_update" + } else { + false + }; + + // Added tests + let added_in_altair = if fork_name < ForkName::Altair { + T::name() != "sync_committee_updates" + && T::name() != "inactivity_updates" + && T::name() != "participation_flag_updates" + && T::name() != "historical_summaries_update" + } else { + false + }; + let added_in_electra = if fork_name < ForkName::Electra { + T::name() != "pending_consolidations" && T::name() != "pending_balance_deposits" + } else { + false + }; + + deprecated_in_altair + && deprecated_in_bellatrix + && deprecated_in_capella + && added_in_altair + && added_in_electra } fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> { From d781420d039c5e8aa1770bf90f27854f58fd241f Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 08:17:14 -0400 Subject: [PATCH 080/227] add consolidations to merkle calc for inclusion proof --- consensus/types/src/beacon_block_body.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index fba542450de..0466d1b7680 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -251,6 +251,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, body.execution_payload.tree_hash_root(), body.bls_to_execution_changes.tree_hash_root(), body.blob_kzg_commitments.tree_hash_root(), + body.consolidations.tree_hash_root(), ]; let beacon_block_body_depth = leaves.len().next_power_of_two().ilog2() as usize; let tree = MerkleTree::create(&leaves, beacon_block_body_depth); From b7d983d655d5caa8d8b8e57072dc8eedefe6a8ac Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Sun, 12 May 2024 15:18:34 +0300 Subject: [PATCH 081/227] fix should_override_fc tests --- testing/ef_tests/src/handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index 2f861af71e4..2e04b5f3b88 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -584,7 +584,7 @@ impl Handler for ForkChoiceHandler { // No FCU override tests prior to bellatrix. if self.handler_name == "should_override_forkchoice_update" - && fork_name >= ForkName::Bellatrix + && fork_name < ForkName::Bellatrix { return false; } From 2460a5960f8dc6549f73b455b767292eeed7120a Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 08:33:18 -0400 Subject: [PATCH 082/227] get ssz tests compiling --- testing/ef_tests/src/type_name.rs | 9 +++++++++ testing/ef_tests/tests/tests.rs | 24 ------------------------ 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index cbea78dabfc..d1a7a442bbd 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -52,25 +52,31 @@ type_name_generic!(BeaconBlockBodyAltair, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyBellatrix, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyCapella, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyDeneb, "BeaconBlockBody"); +type_name_generic!(BeaconBlockBodyElectra, "BeaconBlockBody"); type_name!(BeaconBlockHeader); type_name_generic!(BeaconState); type_name!(BlobIdentifier); type_name_generic!(BlobSidecar); type_name!(Checkpoint); +type_name!(Consolidation); type_name_generic!(ContributionAndProof); type_name!(Deposit); type_name!(DepositData); type_name!(DepositMessage); +type_name!(DepositReceipt); type_name!(Eth1Data); +type_name!(ExecutionLayerWithdrawalRequest); type_name_generic!(ExecutionPayload); type_name_generic!(ExecutionPayloadBellatrix, "ExecutionPayload"); type_name_generic!(ExecutionPayloadCapella, "ExecutionPayload"); type_name_generic!(ExecutionPayloadDeneb, "ExecutionPayload"); +type_name_generic!(ExecutionPayloadElectra, "ExecutionPayload"); type_name_generic!(FullPayload, "ExecutionPayload"); type_name_generic!(ExecutionPayloadHeader); type_name_generic!(ExecutionPayloadHeaderBellatrix, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderCapella, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderDeneb, "ExecutionPayloadHeader"); +type_name_generic!(ExecutionPayloadHeaderElectra, "ExecutionPayloadHeader"); type_name_generic!(BlindedPayload, "ExecutionPayloadHeader"); type_name!(Fork); type_name!(ForkData); @@ -109,6 +115,9 @@ type_name_generic!(LightClientUpdateAltair, "LightClientUpdate"); type_name_generic!(LightClientUpdateCapella, "LightClientUpdate"); type_name_generic!(LightClientUpdateDeneb, "LightClientUpdate"); type_name_generic!(PendingAttestation); +type_name!(PendingConsolidation); +type_name!(PendingPartialWithdrawal); +type_name!(PendingBalanceDeposit); type_name!(ProposerSlashing); type_name_generic!(SignedAggregateAndProof); type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProof"); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index f9879cdbcf9..cb9cffa14c0 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -356,10 +356,6 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only() - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only() - .run(); } // LightClientHeader has no internal indicator of which fork it is for, so we test it separately. @@ -385,12 +381,6 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( - ) - .run(); } // LightClientOptimisticUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -404,8 +394,6 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::capella_only().run(); SszStaticHandler::, MinimalEthSpec>::deneb_only().run(); SszStaticHandler::, MainnetEthSpec>::deneb_only().run(); - SszStaticHandler::, MinimalEthSpec>::electra_only().run(); - SszStaticHandler::, MainnetEthSpec>::electra_only().run(); } // LightClientFinalityUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -435,12 +423,6 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::deneb_only( ) .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( - ) - .run(); } // LightClientUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -464,12 +446,6 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( - ) - .run(); } #[test] From 4094d9a4fbd252c779c948e878089be3e6c59464 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 08:38:21 -0400 Subject: [PATCH 083/227] ssz merkle validity for only deneb --- testing/ef_tests/src/handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index 2e04b5f3b88..daa8904f2ed 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -833,7 +833,7 @@ impl Handler for KzgInclusionMerkleProofValidityHandler bool { // Enabled in Deneb - fork_name >= ForkName::Deneb + fork_name == ForkName::Deneb } } From 172353d8302f88b68d81fff5e9cbcc028719e1e0 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Sun, 12 May 2024 15:57:53 +0300 Subject: [PATCH 084/227] im dumb af --- testing/ef_tests/src/type_name.rs | 2 ++ testing/ef_tests/tests/tests.rs | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index d1a7a442bbd..1280034f7b5 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -82,6 +82,8 @@ type_name!(Fork); type_name!(ForkData); type_name_generic!(HistoricalBatch); type_name_generic!(IndexedAttestation); +type_name_generic!(IndexedAttestationBase, "IndexedAttestation"); +type_name_generic!(IndexedAttestationElectra, "IndexedAttestation"); type_name_generic!(LightClientBootstrap); type_name_generic!(LightClientBootstrapAltair, "LightClientBootstrap"); type_name_generic!(LightClientBootstrapCapella, "LightClientBootstrap"); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index cb9cffa14c0..c14cf53bbdb 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -241,7 +241,6 @@ mod ssz_static { ssz_static_test!(fork, Fork); ssz_static_test!(fork_data, ForkData); ssz_static_test!(historical_batch, HistoricalBatch<_>); - ssz_static_test!(indexed_attestation, IndexedAttestation<_>); ssz_static_test!(pending_attestation, PendingAttestation<_>); ssz_static_test!(proposer_slashing, ProposerSlashing); ssz_static_test!( @@ -267,6 +266,18 @@ mod ssz_static { .run(); } + #[test] + fn indexed_attestation() { + SszStaticHandler::, MinimalEthSpec>::pre_electra() + .run(); + SszStaticHandler::, MainnetEthSpec>::pre_electra() + .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only() + .run(); + } + #[test] fn signed_aggregate_and_proof() { SszStaticHandler::, MinimalEthSpec>::pre_electra( From affdccd460fee510e2ce0f1c995d2507850b3473 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 12:38:39 -0400 Subject: [PATCH 085/227] enable tests --- testing/ef_tests/check_all_files_accessed.py | 4 +- .../ef_tests/src/cases/epoch_processing.rs | 61 ++++++++----------- testing/ef_tests/tests/tests.rs | 20 +++++- 3 files changed, 47 insertions(+), 38 deletions(-) diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index 7629d61827f..2405e19d5f0 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -42,7 +42,9 @@ "bls12-381-tests/deserialization_G2", "bls12-381-tests/hash_to_G2", "tests/.*/eip6110", - "tests/.*/whisk" + "tests/.*/whisk", + "tests/.*/eip7594", + "tests/.*/electra/ssz_static/LightClient*" ] diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index 302c5fa42e8..390dab9a2b1 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -342,44 +342,33 @@ impl> Case for EpochProcessing { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - // Deprecated tests - let deprecated_in_altair = if fork_name > ForkName::Base { - T::name() != "participation_record_updates" - && T::name() != "historical_summaries_update" - } else { - false - }; - let deprecated_in_bellatrix = if fork_name > ForkName::Altair { - T::name() != "historical_roots_update" - } else { - false - }; - let deprecated_in_capella = if fork_name > ForkName::Bellatrix { - T::name() != "historical_summaries_update" - } else { - false - }; + if fork_name > ForkName::Base && (T::name() == "participation_record_updates") { + return false; + } - // Added tests - let added_in_altair = if fork_name < ForkName::Altair { - T::name() != "sync_committee_updates" - && T::name() != "inactivity_updates" - && T::name() != "participation_flag_updates" - && T::name() != "historical_summaries_update" - } else { - false - }; - let added_in_electra = if fork_name < ForkName::Electra { - T::name() != "pending_consolidations" && T::name() != "pending_balance_deposits" - } else { - false - }; + if fork_name > ForkName::Bellatrix && T::name() == "historical_roots_update" { + return false; + } + + if fork_name < ForkName::Capella && T::name() == "historical_summaries_update" { + return false; + } + + if fork_name < ForkName::Altair + && (T::name() == "sync_committee_updates" + || T::name() == "inactivity_updates" + || T::name() == "participation_flag_updates" + || T::name() == "historical_summaries_update") + { + return false; + } - deprecated_in_altair - && deprecated_in_bellatrix - && deprecated_in_capella - && added_in_altair - && added_in_electra + if fork_name < ForkName::Electra + && (T::name() == "pending_consolidations" || T::name() == "pending_balance_deposits") + { + return false; + } + true } fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> { diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index c14cf53bbdb..6720376f8af 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -1,7 +1,7 @@ #![cfg(feature = "ef_tests")] use ef_tests::*; -use types::{MainnetEthSpec, MinimalEthSpec, *}; +use types::{ExecutionLayerWithdrawalRequest, MainnetEthSpec, MinimalEthSpec, *}; // Check that the hand-computed multiplications on EthSpec are correctly computed. // This test lives here because one is most likely to muck these up during a spec update. @@ -92,6 +92,24 @@ fn operations_withdrawals() { OperationsHandler::>::default().run(); } +#[test] +fn operations_execution_layer_withdrawal_reqeusts() { + OperationsHandler::::default().run(); + OperationsHandler::::default().run(); +} + +#[test] +fn operations_deposit_receipts() { + OperationsHandler::::default().run(); + OperationsHandler::::default().run(); +} + +#[test] +fn operations_consolidations() { + OperationsHandler::::default().run(); + OperationsHandler::::default().run(); +} + #[test] fn operations_bls_to_execution_change() { OperationsHandler::::default().run(); From 217fa9f805361be1ad810831c26338a3c41a3bf0 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Sun, 12 May 2024 20:31:20 +0300 Subject: [PATCH 086/227] Fix Consolidation Sigs & Withdrawals --- .../src/per_block_processing/process_operations.rs | 4 ++-- .../src/per_block_processing/signature_sets.rs | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 9d2235e2d43..09c7c540ede 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -559,8 +559,8 @@ pub fn process_execution_layer_withdrawal_requests( let has_correct_credential = validator.has_execution_withdrawal_credential(spec); let is_correct_source_address = validator .get_execution_withdrawal_address(spec) - .ok_or(BeaconStateError::NonExecutionAddresWithdrawalCredential)? - == request.source_address; + .map(|addr| addr == request.source_address) + .unwrap_or(false); if !(has_correct_credential && is_correct_source_address) { continue; diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index 65b14714f86..b8316063a3d 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -681,10 +681,9 @@ where let source_index = consolidation.message.source_index as usize; let target_index = consolidation.message.target_index as usize; - let domain = spec.get_domain( - consolidation.message.epoch, + let domain = spec.compute_domain( Domain::Consolidation, - &state.fork(), + spec.genesis_fork_version, state.genesis_validators_root(), ); From 657887f313119dd87161ce366e6cc21c04f81a99 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 16:05:39 -0400 Subject: [PATCH 087/227] comment out consolidation operation ef tests until they are added --- testing/ef_tests/tests/tests.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 6720376f8af..366bc885a73 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -104,11 +104,12 @@ fn operations_deposit_receipts() { OperationsHandler::::default().run(); } -#[test] -fn operations_consolidations() { - OperationsHandler::::default().run(); - OperationsHandler::::default().run(); -} +//TODO(electra) enable once tests are added for these +// #[test] +// fn operations_consolidations() { +// OperationsHandler::::default().run(); +// OperationsHandler::::default().run(); +// } #[test] fn operations_bls_to_execution_change() { From 9c33a6ff9e542032aec39c665c2db100141247d2 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 20:30:55 -0400 Subject: [PATCH 088/227] add signed consolidation ssz tests --- testing/ef_tests/src/type_name.rs | 1 + testing/ef_tests/tests/tests.rs | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 1280034f7b5..6141fd902c6 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -126,6 +126,7 @@ type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProof"); type_name_generic!(SignedAggregateAndProofElectra, "SignedAggregateAndProof"); type_name_generic!(SignedBeaconBlock); type_name!(SignedBeaconBlockHeader); +type_name!(SignedConsolidation); type_name_generic!(SignedContributionAndProof); type_name!(SignedVoluntaryExit); type_name!(SigningData); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 366bc885a73..8b46823404b 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -628,6 +628,12 @@ mod ssz_static { SszStaticHandler::::electra_and_later().run(); SszStaticHandler::::electra_and_later().run(); } + + #[test] + fn signed_consolidation() { + SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); + } } #[test] From 75f6f7810a2b791cadcef6f6e3583fe3a441e3ac Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 13 May 2024 02:37:51 -0400 Subject: [PATCH 089/227] add as_iter to all List and Vector fields in the beacon state --- .../src/per_block_processing/process_operations.rs | 3 +++ consensus/types/src/beacon_state.rs | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 09c7c540ede..8047f677849 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -476,7 +476,10 @@ pub fn apply_deposit( // The signature should be checked for new validators. Return early for a bad // signature. if is_valid_deposit_signature(&deposit_data, spec).is_err() { + dbg!("invalid sig"); return Ok(()); + } else { + dbg!("valid sig"); } let new_validator_index = state.validators().len(); diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 6b4bfd29cb5..f2fdbc27802 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -371,6 +371,7 @@ where pub eth1_deposit_index: u64, // Registry + #[compare_fields(as_iter)] #[test_random(default)] pub validators: List, #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] @@ -396,8 +397,10 @@ where pub current_epoch_attestations: List, E::MaxPendingAttestations>, // Participation (Altair and later) + #[compare_fields(as_iter)] #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] #[test_random(default)] + #[compare_fields(as_iter)] pub previous_epoch_participation: List, #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] #[test_random(default)] @@ -494,13 +497,16 @@ where #[superstruct(only(Electra), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] pub earliest_consolidation_epoch: Epoch, + #[compare_fields(as_iter)] #[test_random(default)] #[superstruct(only(Electra))] pub pending_balance_deposits: List, + #[compare_fields(as_iter)] #[test_random(default)] #[superstruct(only(Electra))] pub pending_partial_withdrawals: List, + #[compare_fields(as_iter)] #[test_random(default)] #[superstruct(only(Electra))] pub pending_consolidations: List, From ac2e03b87a23fa680f9c0d105c21146ccbc23c6a Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 13 May 2024 02:38:35 -0400 Subject: [PATCH 090/227] remove dbg --- .../src/per_block_processing/process_operations.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 8047f677849..09c7c540ede 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -476,10 +476,7 @@ pub fn apply_deposit( // The signature should be checked for new validators. Return early for a bad // signature. if is_valid_deposit_signature(&deposit_data, spec).is_err() { - dbg!("invalid sig"); return Ok(()); - } else { - dbg!("valid sig"); } let new_validator_index = state.validators().len(); From c680164742824b8f7c4da8943ef644518817b58c Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Mon, 13 May 2024 10:18:17 +0300 Subject: [PATCH 091/227] Send unagg attestation based on fork --- validator_client/src/attestation_service.rs | 33 +++++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 062a9436e95..4eff170c229 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -14,11 +14,11 @@ use std::ops::Deref; use std::sync::Arc; use tokio::time::{sleep, sleep_until, Duration, Instant}; use tree_hash::TreeHash; -use types::ForkName; use types::{ attestation::AttestationBase, AggregateSignature, Attestation, AttestationData, BitList, ChainSpec, CommitteeIndex, EthSpec, Slot, }; +use types::{AttestationElectra, BitVector, ForkName}; /// Builds an `AttestationService`. pub struct AttestationServiceBuilder { @@ -383,11 +383,32 @@ impl AttestationService { return None; } - let mut attestation = Attestation::Base(AttestationBase { - aggregation_bits: BitList::with_capacity(duty.committee_length as usize).unwrap(), - data: attestation_data.clone(), - signature: AggregateSignature::infinity(), - }); + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); + + let mut attestation = if fork_name >= ForkName::Electra { + let mut committee_bits: BitVector = BitVector::default(); + committee_bits + .set(duty.committee_index as usize, true) + .unwrap(); + Attestation::Electra(AttestationElectra { + aggregation_bits: BitList::with_capacity(duty.committee_length as usize) + .unwrap(), + data: attestation_data.clone(), + committee_bits, + signature: AggregateSignature::infinity(), + }) + } else { + Attestation::Base(AttestationBase { + aggregation_bits: BitList::with_capacity(duty.committee_length as usize) + .unwrap(), + data: attestation_data.clone(), + signature: AggregateSignature::infinity(), + }) + }; match self .validator_store From d9200a983781ae1b54b7e4adc7f3ce3c64fe655c Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Mon, 13 May 2024 11:13:49 +0300 Subject: [PATCH 092/227] Fix Epoch Processing Consolidations --- .../src/per_epoch_processing/single_pass.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/consensus/state_processing/src/per_epoch_processing/single_pass.rs b/consensus/state_processing/src/per_epoch_processing/single_pass.rs index f5d3f974a6e..4ab756b561b 100644 --- a/consensus/state_processing/src/per_epoch_processing/single_pass.rs +++ b/consensus/state_processing/src/per_epoch_processing/single_pass.rs @@ -359,6 +359,7 @@ pub fn process_epoch_single_pass( state, &mut next_epoch_cache, effective_balances_ctxt, + conf.effective_balance_updates, state_ctxt, spec, )?; @@ -859,6 +860,7 @@ fn process_pending_consolidations( state: &mut BeaconState, next_epoch_cache: &mut PreEpochCache, effective_balances_ctxt: &EffectiveBalancesContext, + perform_effective_balance_updates: bool, state_ctxt: &StateContext, spec: &ChainSpec, ) -> Result<(), Error> { @@ -910,6 +912,11 @@ fn process_pending_consolidations( )?; *state.pending_consolidations_mut()? = new_pending_consolidations; + // the spec tests require we don't perform effective balance updates when testing pending_consolidations + if !perform_effective_balance_updates { + return Ok(()); + } + // Re-process effective balance updates for validators affected by consolidations. let (validators, balances, _, current_epoch_participation, _, progressive_balances, _, _) = state.mutable_validator_fields()?; From 7e57ab3af2acb6639ea00b05e5f1b4dd2d65f8f6 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Mon, 13 May 2024 11:36:35 +0300 Subject: [PATCH 093/227] Fix Final Spec Tests! --- testing/ef_tests/tests/tests.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 8b46823404b..ab033b13425 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -99,17 +99,18 @@ fn operations_execution_layer_withdrawal_reqeusts() { } #[test] +#[cfg(not(feature = "fake_crypto"))] fn operations_deposit_receipts() { OperationsHandler::::default().run(); OperationsHandler::::default().run(); } -//TODO(electra) enable once tests are added for these -// #[test] -// fn operations_consolidations() { -// OperationsHandler::::default().run(); -// OperationsHandler::::default().run(); -// } +#[test] +fn operations_consolidations() { + OperationsHandler::::default().run(); + //TODO(electra): re-enable mainnet once they make tests for this + //OperationsHandler::::default().run(); +} #[test] fn operations_bls_to_execution_change() { From 7f5490675cb1a83a8f52ab0d250a93224daeb8b4 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 14 May 2024 15:03:52 +0300 Subject: [PATCH 094/227] Fix ser/de --- .../execution_layer/src/engine_api/json_structures.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 28eb90fd1fa..034483eb73b 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -838,10 +838,10 @@ impl TryFrom for ClientVersionV1 { pub struct JsonDepositRequest { pub pubkey: PublicKeyBytes, pub withdrawal_credentials: Hash256, - #[serde(with = "serde_utils::quoted_u64")] + #[serde(with = "serde_utils::u64_hex_be")] pub amount: u64, pub signature: Signature, - #[serde(with = "serde_utils::quoted_u64")] + #[serde(with = "serde_utils::u64_hex_be")] pub index: u64, } @@ -874,7 +874,7 @@ impl From for DepositReceipt { pub struct JsonWithdrawalRequest { pub source_address: Address, pub validator_public_key: PublicKeyBytes, - #[serde(with = "serde_utils::quoted_u64")] + #[serde(with = "serde_utils::u64_hex_be")] pub amount: u64, } From dfcce10a81e9ee31486d473c95b8278a3995bf58 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 15 May 2024 08:37:20 +0300 Subject: [PATCH 095/227] Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra --- .../beacon_chain/src/attestation_verification.rs | 6 +++--- beacon_node/beacon_chain/src/test_utils.rs | 4 ++-- .../beacon_chain/tests/attestation_verification.rs | 4 ++-- consensus/types/src/subnet_id.rs | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index c9742d5468f..4c557fa091b 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -816,9 +816,9 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { - let expected_subnet_id = SubnetId::compute_subnet_for_attestation_data::( - indexed_attestation.data(), + ) -> Result<(u64, SubnetId), Error> { + let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation, committees_per_slot, &chain.spec, ) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index bcf7582ebfd..77bdf7bcc2c 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1168,8 +1168,8 @@ where agg_sig }; - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation.to_ref(), committee_count, &self.chain.spec, ) diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index d132b92a5c2..e91e8c77a3c 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -146,8 +146,8 @@ fn get_valid_unaggregated_attestation( ) .expect("should sign attestation"); - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - valid_attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &valid_attestation.to_ref(), head.beacon_state .get_committee_count_at_slot(current_slot) .expect("should get committee count"), diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index 9b6a2e6a192..ec447956749 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -1,5 +1,5 @@ //! Identifies each shard by an integer identifier. -use crate::{AttestationData, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; +use crate::{AttestationRef, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; @@ -37,16 +37,16 @@ impl SubnetId { id.into() } - /// Compute the subnet for an attestation with `attestation_data` where each slot in the + /// Compute the subnet for an attestation where each slot in the /// attestation epoch contains `committee_count_per_slot` committees. - pub fn compute_subnet_for_attestation_data( - attestation_data: &AttestationData, + pub fn compute_subnet_for_attestation( + attestation: &AttestationRef, committee_count_per_slot: u64, spec: &ChainSpec, ) -> Result { Self::compute_subnet::( - attestation_data.slot, - attestation_data.index, + attestation.data().slot, + attestation.committee_index(), committee_count_per_slot, spec, ) From 01dc321c82514accac3dcc392265eeb9cf6f6730 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 May 2024 02:52:10 -0400 Subject: [PATCH 096/227] cargo fmt --- beacon_node/beacon_chain/src/attestation_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 4c557fa091b..5c144fb3a00 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -816,7 +816,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { + ) -> Result<(u64, SubnetId), Error> { let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( &attestation, committees_per_slot, From f1fb03307eaaf642106f6f077c8075389e22424d Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 15 May 2024 08:37:20 +0300 Subject: [PATCH 097/227] Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra --- .../beacon_chain/src/attestation_verification.rs | 6 +++--- beacon_node/beacon_chain/src/test_utils.rs | 4 ++-- .../beacon_chain/tests/attestation_verification.rs | 4 ++-- consensus/types/src/subnet_id.rs | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index c9742d5468f..4c557fa091b 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -816,9 +816,9 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { - let expected_subnet_id = SubnetId::compute_subnet_for_attestation_data::( - indexed_attestation.data(), + ) -> Result<(u64, SubnetId), Error> { + let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation, committees_per_slot, &chain.spec, ) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index bcf7582ebfd..77bdf7bcc2c 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1168,8 +1168,8 @@ where agg_sig }; - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation.to_ref(), committee_count, &self.chain.spec, ) diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index d132b92a5c2..e91e8c77a3c 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -146,8 +146,8 @@ fn get_valid_unaggregated_attestation( ) .expect("should sign attestation"); - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - valid_attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &valid_attestation.to_ref(), head.beacon_state .get_committee_count_at_slot(current_slot) .expect("should get committee count"), diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index 9b6a2e6a192..ec447956749 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -1,5 +1,5 @@ //! Identifies each shard by an integer identifier. -use crate::{AttestationData, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; +use crate::{AttestationRef, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; @@ -37,16 +37,16 @@ impl SubnetId { id.into() } - /// Compute the subnet for an attestation with `attestation_data` where each slot in the + /// Compute the subnet for an attestation where each slot in the /// attestation epoch contains `committee_count_per_slot` committees. - pub fn compute_subnet_for_attestation_data( - attestation_data: &AttestationData, + pub fn compute_subnet_for_attestation( + attestation: &AttestationRef, committee_count_per_slot: u64, spec: &ChainSpec, ) -> Result { Self::compute_subnet::( - attestation_data.slot, - attestation_data.index, + attestation.data().slot, + attestation.committee_index(), committee_count_per_slot, spec, ) From 2629393ad92bfe005fc1c6bf558c1d7a586fd03e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 May 2024 02:52:10 -0400 Subject: [PATCH 098/227] cargo fmt --- beacon_node/beacon_chain/src/attestation_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 4c557fa091b..5c144fb3a00 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -816,7 +816,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { + ) -> Result<(u64, SubnetId), Error> { let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( &attestation, committees_per_slot, From 8506fb056f86e38df416e6ce782d770fa50385d4 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 15 May 2024 08:37:20 +0300 Subject: [PATCH 099/227] Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra --- .../beacon_chain/src/attestation_verification.rs | 6 +++--- beacon_node/beacon_chain/src/test_utils.rs | 4 ++-- .../beacon_chain/tests/attestation_verification.rs | 4 ++-- consensus/types/src/subnet_id.rs | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3d722a534be..3edb7af2a9c 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -795,9 +795,9 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { - let expected_subnet_id = SubnetId::compute_subnet_for_attestation_data::( - indexed_attestation.data(), + ) -> Result<(u64, SubnetId), Error> { + let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation, committees_per_slot, &chain.spec, ) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index bcf7582ebfd..77bdf7bcc2c 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1168,8 +1168,8 @@ where agg_sig }; - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation.to_ref(), committee_count, &self.chain.spec, ) diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index d132b92a5c2..e91e8c77a3c 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -146,8 +146,8 @@ fn get_valid_unaggregated_attestation( ) .expect("should sign attestation"); - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - valid_attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &valid_attestation.to_ref(), head.beacon_state .get_committee_count_at_slot(current_slot) .expect("should get committee count"), diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index 9b6a2e6a192..ec447956749 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -1,5 +1,5 @@ //! Identifies each shard by an integer identifier. -use crate::{AttestationData, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; +use crate::{AttestationRef, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; @@ -37,16 +37,16 @@ impl SubnetId { id.into() } - /// Compute the subnet for an attestation with `attestation_data` where each slot in the + /// Compute the subnet for an attestation where each slot in the /// attestation epoch contains `committee_count_per_slot` committees. - pub fn compute_subnet_for_attestation_data( - attestation_data: &AttestationData, + pub fn compute_subnet_for_attestation( + attestation: &AttestationRef, committee_count_per_slot: u64, spec: &ChainSpec, ) -> Result { Self::compute_subnet::( - attestation_data.slot, - attestation_data.index, + attestation.data().slot, + attestation.committee_index(), committee_count_per_slot, spec, ) From ec055f4717f289066dc0ef6b09fe0a93e5306c8c Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 May 2024 02:52:10 -0400 Subject: [PATCH 100/227] cargo fmt --- beacon_node/beacon_chain/src/attestation_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3edb7af2a9c..9c44c5529b3 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -795,7 +795,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { + ) -> Result<(u64, SubnetId), Error> { let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( &attestation, committees_per_slot, From 4f0ecf2a5cf228dea434bba0554c79300a394479 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 15 May 2024 08:37:20 +0300 Subject: [PATCH 101/227] Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra --- .../beacon_chain/src/attestation_verification.rs | 6 +++--- beacon_node/beacon_chain/src/test_utils.rs | 4 ++-- .../beacon_chain/tests/attestation_verification.rs | 4 ++-- consensus/types/src/subnet_id.rs | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3d722a534be..3edb7af2a9c 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -795,9 +795,9 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { - let expected_subnet_id = SubnetId::compute_subnet_for_attestation_data::( - indexed_attestation.data(), + ) -> Result<(u64, SubnetId), Error> { + let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation, committees_per_slot, &chain.spec, ) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index bcf7582ebfd..77bdf7bcc2c 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1168,8 +1168,8 @@ where agg_sig }; - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation.to_ref(), committee_count, &self.chain.spec, ) diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index d132b92a5c2..e91e8c77a3c 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -146,8 +146,8 @@ fn get_valid_unaggregated_attestation( ) .expect("should sign attestation"); - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - valid_attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &valid_attestation.to_ref(), head.beacon_state .get_committee_count_at_slot(current_slot) .expect("should get committee count"), diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index 9b6a2e6a192..ec447956749 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -1,5 +1,5 @@ //! Identifies each shard by an integer identifier. -use crate::{AttestationData, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; +use crate::{AttestationRef, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; @@ -37,16 +37,16 @@ impl SubnetId { id.into() } - /// Compute the subnet for an attestation with `attestation_data` where each slot in the + /// Compute the subnet for an attestation where each slot in the /// attestation epoch contains `committee_count_per_slot` committees. - pub fn compute_subnet_for_attestation_data( - attestation_data: &AttestationData, + pub fn compute_subnet_for_attestation( + attestation: &AttestationRef, committee_count_per_slot: u64, spec: &ChainSpec, ) -> Result { Self::compute_subnet::( - attestation_data.slot, - attestation_data.index, + attestation.data().slot, + attestation.committee_index(), committee_count_per_slot, spec, ) From 227aa4bc4f609da683df8f91ae56ae3672aba2ce Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 May 2024 02:52:10 -0400 Subject: [PATCH 102/227] cargo fmt --- beacon_node/beacon_chain/src/attestation_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3edb7af2a9c..9c44c5529b3 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -795,7 +795,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { + ) -> Result<(u64, SubnetId), Error> { let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( &attestation, committees_per_slot, From fc15736fcb4022699c9cdb8da0cda81e7bdf6c46 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 15 May 2024 08:37:20 +0300 Subject: [PATCH 103/227] Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra --- .../beacon_chain/src/attestation_verification.rs | 6 +++--- beacon_node/beacon_chain/src/test_utils.rs | 4 ++-- .../beacon_chain/tests/attestation_verification.rs | 4 ++-- consensus/types/src/subnet_id.rs | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3d722a534be..3edb7af2a9c 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -795,9 +795,9 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { - let expected_subnet_id = SubnetId::compute_subnet_for_attestation_data::( - indexed_attestation.data(), + ) -> Result<(u64, SubnetId), Error> { + let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation, committees_per_slot, &chain.spec, ) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index bcf7582ebfd..77bdf7bcc2c 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1168,8 +1168,8 @@ where agg_sig }; - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation.to_ref(), committee_count, &self.chain.spec, ) diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index d132b92a5c2..e91e8c77a3c 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -146,8 +146,8 @@ fn get_valid_unaggregated_attestation( ) .expect("should sign attestation"); - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - valid_attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &valid_attestation.to_ref(), head.beacon_state .get_committee_count_at_slot(current_slot) .expect("should get committee count"), diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index 9b6a2e6a192..ec447956749 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -1,5 +1,5 @@ //! Identifies each shard by an integer identifier. -use crate::{AttestationData, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; +use crate::{AttestationRef, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; @@ -37,16 +37,16 @@ impl SubnetId { id.into() } - /// Compute the subnet for an attestation with `attestation_data` where each slot in the + /// Compute the subnet for an attestation where each slot in the /// attestation epoch contains `committee_count_per_slot` committees. - pub fn compute_subnet_for_attestation_data( - attestation_data: &AttestationData, + pub fn compute_subnet_for_attestation( + attestation: &AttestationRef, committee_count_per_slot: u64, spec: &ChainSpec, ) -> Result { Self::compute_subnet::( - attestation_data.slot, - attestation_data.index, + attestation.data().slot, + attestation.committee_index(), committee_count_per_slot, spec, ) From d8941d70b679f3e36c25e143e578c4d6c141664a Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 May 2024 02:52:10 -0400 Subject: [PATCH 104/227] cargo fmt --- beacon_node/beacon_chain/src/attestation_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3edb7af2a9c..9c44c5529b3 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -795,7 +795,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { + ) -> Result<(u64, SubnetId), Error> { let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( &attestation, committees_per_slot, From 0c29896438b8d5039a605e52ad1c9120aedfa24b Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 15 May 2024 08:37:20 +0300 Subject: [PATCH 105/227] Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra --- .../beacon_chain/src/attestation_verification.rs | 6 +++--- beacon_node/beacon_chain/src/test_utils.rs | 4 ++-- .../beacon_chain/tests/attestation_verification.rs | 4 ++-- consensus/types/src/subnet_id.rs | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3d722a534be..3edb7af2a9c 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -795,9 +795,9 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { - let expected_subnet_id = SubnetId::compute_subnet_for_attestation_data::( - indexed_attestation.data(), + ) -> Result<(u64, SubnetId), Error> { + let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation, committees_per_slot, &chain.spec, ) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index bcf7582ebfd..77bdf7bcc2c 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1168,8 +1168,8 @@ where agg_sig }; - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation.to_ref(), committee_count, &self.chain.spec, ) diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index d132b92a5c2..e91e8c77a3c 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -146,8 +146,8 @@ fn get_valid_unaggregated_attestation( ) .expect("should sign attestation"); - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - valid_attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &valid_attestation.to_ref(), head.beacon_state .get_committee_count_at_slot(current_slot) .expect("should get committee count"), diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index 9b6a2e6a192..ec447956749 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -1,5 +1,5 @@ //! Identifies each shard by an integer identifier. -use crate::{AttestationData, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; +use crate::{AttestationRef, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; @@ -37,16 +37,16 @@ impl SubnetId { id.into() } - /// Compute the subnet for an attestation with `attestation_data` where each slot in the + /// Compute the subnet for an attestation where each slot in the /// attestation epoch contains `committee_count_per_slot` committees. - pub fn compute_subnet_for_attestation_data( - attestation_data: &AttestationData, + pub fn compute_subnet_for_attestation( + attestation: &AttestationRef, committee_count_per_slot: u64, spec: &ChainSpec, ) -> Result { Self::compute_subnet::( - attestation_data.slot, - attestation_data.index, + attestation.data().slot, + attestation.committee_index(), committee_count_per_slot, spec, ) From 210ad2ff778449ae2e6f94b1ac0e0d712a2ba053 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 May 2024 02:52:10 -0400 Subject: [PATCH 106/227] cargo fmt --- beacon_node/beacon_chain/src/attestation_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3edb7af2a9c..9c44c5529b3 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -795,7 +795,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { + ) -> Result<(u64, SubnetId), Error> { let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( &attestation, committees_per_slot, From 79a5f2556f78069cdc382660f3de51283ad4e7f1 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 15 May 2024 08:37:20 +0300 Subject: [PATCH 107/227] Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra --- .../beacon_chain/src/attestation_verification.rs | 6 +++--- beacon_node/beacon_chain/src/test_utils.rs | 4 ++-- .../beacon_chain/tests/attestation_verification.rs | 4 ++-- consensus/types/src/subnet_id.rs | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3d722a534be..3edb7af2a9c 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -795,9 +795,9 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { - let expected_subnet_id = SubnetId::compute_subnet_for_attestation_data::( - indexed_attestation.data(), + ) -> Result<(u64, SubnetId), Error> { + let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation, committees_per_slot, &chain.spec, ) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index bcf7582ebfd..77bdf7bcc2c 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1168,8 +1168,8 @@ where agg_sig }; - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &attestation.to_ref(), committee_count, &self.chain.spec, ) diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index d132b92a5c2..e91e8c77a3c 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -146,8 +146,8 @@ fn get_valid_unaggregated_attestation( ) .expect("should sign attestation"); - let subnet_id = SubnetId::compute_subnet_for_attestation_data::( - valid_attestation.data(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + &valid_attestation.to_ref(), head.beacon_state .get_committee_count_at_slot(current_slot) .expect("should get committee count"), diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index 9b6a2e6a192..ec447956749 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -1,5 +1,5 @@ //! Identifies each shard by an integer identifier. -use crate::{AttestationData, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; +use crate::{AttestationRef, ChainSpec, CommitteeIndex, Epoch, EthSpec, Slot}; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; @@ -37,16 +37,16 @@ impl SubnetId { id.into() } - /// Compute the subnet for an attestation with `attestation_data` where each slot in the + /// Compute the subnet for an attestation where each slot in the /// attestation epoch contains `committee_count_per_slot` committees. - pub fn compute_subnet_for_attestation_data( - attestation_data: &AttestationData, + pub fn compute_subnet_for_attestation( + attestation: &AttestationRef, committee_count_per_slot: u64, spec: &ChainSpec, ) -> Result { Self::compute_subnet::( - attestation_data.slot, - attestation_data.index, + attestation.data().slot, + attestation.committee_index(), committee_count_per_slot, spec, ) From a8088f1bfa57b906d03e6eb044a1e97b54f26d9e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 May 2024 02:52:10 -0400 Subject: [PATCH 108/227] cargo fmt --- beacon_node/beacon_chain/src/attestation_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 3edb7af2a9c..9c44c5529b3 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -795,7 +795,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { committees_per_slot: u64, subnet_id: Option, chain: &BeaconChain, - ) -> Result<(u64, SubnetId), Error> { + ) -> Result<(u64, SubnetId), Error> { let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( &attestation, committees_per_slot, From 8e537d139ebbbb49269c8baa92aef5fd9e3dc767 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 May 2024 04:35:09 -0400 Subject: [PATCH 109/227] update electra readiness with new endpoints --- .../beacon_chain/src/electra_readiness.rs | 24 +++++++------------ beacon_node/execution_layer/src/engine_api.rs | 2 +- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/beacon_node/beacon_chain/src/electra_readiness.rs b/beacon_node/beacon_chain/src/electra_readiness.rs index 42ee743fe6b..669662cc945 100644 --- a/beacon_node/beacon_chain/src/electra_readiness.rs +++ b/beacon_node/beacon_chain/src/electra_readiness.rs @@ -2,9 +2,7 @@ //! transition. use crate::{BeaconChain, BeaconChainTypes}; -use execution_layer::http::{ - ENGINE_FORKCHOICE_UPDATED_V3, ENGINE_GET_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V3, -}; +use execution_layer::http::{ENGINE_GET_PAYLOAD_V4, ENGINE_NEW_PAYLOAD_V4}; use serde::{Deserialize, Serialize}; use std::fmt; use std::time::Duration; @@ -22,7 +20,7 @@ pub enum ElectraReadiness { /// The execution engine is electra-enabled (as far as we can tell) Ready, /// We are connected to an execution engine which doesn't support the V3 engine api methods - V3MethodsNotSupported { error: String }, + V4MethodsNotSupported { error: String }, /// The transition configuration with the EL failed, there might be a problem with /// connectivity, authentication or a difference in configuration. ExchangeCapabilitiesFailed { error: String }, @@ -47,7 +45,7 @@ impl fmt::Display for ElectraReadiness { "The --execution-endpoint flag is not specified, this is a \ requirement post-merge" ), - ElectraReadiness::V3MethodsNotSupported { error } => write!( + ElectraReadiness::V4MethodsNotSupported { error } => write!( f, "Execution endpoint does not support Electra methods: {}", error @@ -88,29 +86,23 @@ impl BeaconChain { } } Ok(capabilities) => { - // TODO(electra): Update in the event we get V4s. let mut missing_methods = String::from("Required Methods Unsupported:"); let mut all_good = true; - if !capabilities.get_payload_v3 { + if !capabilities.get_payload_v4 { missing_methods.push(' '); - missing_methods.push_str(ENGINE_GET_PAYLOAD_V3); + missing_methods.push_str(ENGINE_GET_PAYLOAD_V4); all_good = false; } - if !capabilities.forkchoice_updated_v3 { + if !capabilities.new_payload_v4 { missing_methods.push(' '); - missing_methods.push_str(ENGINE_FORKCHOICE_UPDATED_V3); - all_good = false; - } - if !capabilities.new_payload_v3 { - missing_methods.push(' '); - missing_methods.push_str(ENGINE_NEW_PAYLOAD_V3); + missing_methods.push_str(ENGINE_NEW_PAYLOAD_V4); all_good = false; } if all_good { ElectraReadiness::Ready } else { - ElectraReadiness::V3MethodsNotSupported { + ElectraReadiness::V4MethodsNotSupported { error: missing_methods, } } diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 09e36be41f4..6bfd9997f43 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -644,7 +644,7 @@ impl ExecutionPayloadBodyV1 { "block {} is post-electra but payload body doesn't have withdrawals/deposit_receipts/withdrawal_requests \ Check that ELs are returning receipts and withdrawal_requests in getPayloadBody requests", header.block_hash - )) + )); }; Ok(ExecutionPayload::Electra(ExecutionPayloadElectra { parent_hash: header.parent_hash, From 37bcba2f2a11b41534205d8fc5d505d0f048e7a6 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 May 2024 08:36:24 -0400 Subject: [PATCH 110/227] don't block mev boost till genesis finalization --- beacon_node/beacon_chain/src/beacon_chain.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index cbae123a77c..1be5ba3084d 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -6496,9 +6496,8 @@ impl BeaconChain { /// account the current slot when accounting for skips. pub fn is_healthy(&self, parent_root: &Hash256) -> Result { let cached_head = self.canonical_head.cached_head(); - // Check if the merge has been finalized. - if let Some(finalized_hash) = cached_head.forkchoice_update_parameters().finalized_hash { - if ExecutionBlockHash::zero() == finalized_hash { + if let Some(head_hash) = cached_head.forkchoice_update_parameters().head_hash { + if ExecutionBlockHash::zero() == head_hash { return Ok(ChainHealth::PreMerge); } } else { From a2c7838c0351eff68d02f065990a80fe42430157 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Wed, 15 May 2024 15:50:38 +0300 Subject: [PATCH 111/227] Publish all aggregates --- validator_client/src/attestation_service.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 4eff170c229..f005540bc6a 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -367,9 +367,15 @@ impl AttestationService { let duty = &duty_and_proof.duty; let attestation_data = attestation_data_ref; + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); + // Ensure that the attestation matches the duties. #[allow(clippy::suspicious_operation_groupings)] - if duty.slot != attestation_data.slot || duty.committee_index != attestation_data.index + if duty.slot != attestation_data.slot || (fork_name < ForkName::Electra && duty.committee_index != attestation_data.index) { crit!( log, @@ -383,12 +389,6 @@ impl AttestationService { return None; } - let fork_name = self - .context - .eth2_config - .spec - .fork_name_at_slot::(attestation_data.slot); - let mut attestation = if fork_name >= ForkName::Electra { let mut committee_bits: BitVector = BitVector::default(); committee_bits From b5333b4d92b60cce8cd457576148ec550b962461 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Wed, 15 May 2024 17:05:40 +0300 Subject: [PATCH 112/227] just one more check bro plz.. --- validator_client/src/attestation_service.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index f005540bc6a..c8857637179 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -375,7 +375,8 @@ impl AttestationService { // Ensure that the attestation matches the duties. #[allow(clippy::suspicious_operation_groupings)] - if duty.slot != attestation_data.slot || (fork_name < ForkName::Electra && duty.committee_index != attestation_data.index) + if duty.slot != attestation_data.slot + || (fork_name < ForkName::Electra && duty.committee_index != attestation_data.index) { crit!( log, @@ -576,6 +577,12 @@ impl AttestationService { .await .map_err(|e| e.to_string())?; + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); + // Create futures to produce the signed aggregated attestations. let signing_futures = validator_duties.iter().map(|duty_and_proof| async move { let duty = &duty_and_proof.duty; @@ -584,7 +591,9 @@ impl AttestationService { let slot = attestation_data.slot; let committee_index = attestation_data.index; - if duty.slot != slot || duty.committee_index != committee_index { + if duty.slot != slot + || (fork_name < ForkName::Electra && duty.committee_index != committee_index) + { crit!(log, "Inconsistent validator duties during signing"); return None; } From d197a19b571abcec50b3325c816450b0de76085e Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Thu, 16 May 2024 12:55:03 +0300 Subject: [PATCH 113/227] Fix Bug In Block Processing with 0x02 Credentials --- .../state_processing/src/per_block_processing.rs | 4 ++-- consensus/types/src/validator.rs | 12 ------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 0f67dd95f85..05621d188be 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -566,7 +566,7 @@ pub fn get_expected_withdrawals( index: withdrawal_index, validator_index, address: validator - .get_eth1_withdrawal_address(spec) + .get_execution_withdrawal_address(spec) .ok_or(BlockProcessingError::WithdrawalCredentialsInvalid)?, amount: balance, }); @@ -576,7 +576,7 @@ pub fn get_expected_withdrawals( index: withdrawal_index, validator_index, address: validator - .get_eth1_withdrawal_address(spec) + .get_execution_withdrawal_address(spec) .ok_or(BlockProcessingError::WithdrawalCredentialsInvalid)?, amount: balance.safe_sub( validator diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index 82d0ad311bb..b5753a6b94b 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -140,18 +140,6 @@ impl Validator { is_compounding_withdrawal_credential(self.withdrawal_credentials, spec) } - /// Get the eth1 withdrawal address if this validator has one initialized. - pub fn get_eth1_withdrawal_address(&self, spec: &ChainSpec) -> Option
{ - self.has_eth1_withdrawal_credential(spec) - .then(|| { - self.withdrawal_credentials - .as_bytes() - .get(12..) - .map(Address::from_slice) - }) - .flatten() - } - /// Get the execution withdrawal address if this validator has one initialized. pub fn get_execution_withdrawal_address(&self, spec: &ChainSpec) -> Option
{ self.has_execution_withdrawal_credential(spec) From bafb5f0cc0aaea9a1ccae7337ae2d371fda41449 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 17 May 2024 09:51:32 -0400 Subject: [PATCH 114/227] fix slashing handling --- consensus/types/src/indexed_attestation.rs | 16 +++++ slasher/src/array.rs | 20 +++++- slasher/src/lib.rs | 81 +++++++++++----------- slasher/src/slasher.rs | 28 +++++--- 4 files changed, 95 insertions(+), 50 deletions(-) diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index 4e7e061b45f..f5b55f6d9b2 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -115,6 +115,22 @@ impl IndexedAttestation { IndexedAttestation::Electra(att) => att.attesting_indices.first(), } } + + pub fn to_electra(self) -> Result, ssz_types::Error> { + Ok(match self { + Self::Base(att) => { + let extended_attesting_indices: VariableList = + VariableList::new(att.attesting_indices.to_vec())?; + + IndexedAttestationElectra { + attesting_indices: extended_attesting_indices, + data: att.data, + signature: att.signature, + } + } + Self::Electra(att) => att, + }) + } } impl<'a, E: EthSpec> IndexedAttestationRef<'a, E> { diff --git a/slasher/src/array.rs b/slasher/src/array.rs index 77ddceb85fe..714ccf4e77b 100644 --- a/slasher/src/array.rs +++ b/slasher/src/array.rs @@ -5,6 +5,7 @@ use crate::{ }; use flate2::bufread::{ZlibDecoder, ZlibEncoder}; use serde::{Deserialize, Serialize}; +use slog::Logger; use std::borrow::Borrow; use std::collections::{btree_map::Entry, BTreeMap, HashSet}; use std::io::Read; @@ -485,6 +486,7 @@ pub fn update( batch: Vec>>, current_epoch: Epoch, config: &Config, + log: &Logger, ) -> Result>, Error> { // Split the batch up into horizontal segments. // Map chunk indexes in the range `0..self.config.chunk_size` to attestations @@ -504,6 +506,7 @@ pub fn update( &chunk_attestations, current_epoch, config, + log, )?; slashings.extend(update_array::<_, MaxTargetChunk>( db, @@ -512,6 +515,7 @@ pub fn update( &chunk_attestations, current_epoch, config, + log, )?); // Update all current epochs. @@ -570,6 +574,7 @@ pub fn update_array( chunk_attestations: &BTreeMap>>>, current_epoch: Epoch, config: &Config, + log: &Logger, ) -> Result>, Error> { let mut slashings = HashSet::new(); // Map from chunk index to updated chunk at that index. @@ -603,8 +608,19 @@ pub fn update_array( current_epoch, config, )?; - if let Some(slashing) = slashing_status.into_slashing(&attestation.indexed) { - slashings.insert(slashing); + match slashing_status.into_slashing(&attestation.indexed) { + Ok(Some(slashing)) => { + slashings.insert(slashing); + } + Err(e) => { + slog::error!( + log, + "Invalid slashing conversion"; + "validator_index" => validator_index, + "error" => e + ); + } + Ok(None) => {} } } } diff --git a/slasher/src/lib.rs b/slasher/src/lib.rs index 2577829577e..339ca1f7d3a 100644 --- a/slasher/src/lib.rs +++ b/slasher/src/lib.rs @@ -53,55 +53,56 @@ impl AttesterSlashingStatus { pub fn into_slashing( self, new_attestation: &IndexedAttestation, - ) -> Option> { + ) -> Result>, String> { use AttesterSlashingStatus::*; // The surrounding attestation must be in `attestation_1` to be valid. - match self { + Ok(match self { NotSlashable => None, AlreadyDoubleVoted => None, - DoubleVote(existing) | SurroundedByExisting(existing) => { - match (*existing, new_attestation) { - // TODO(electra) - determine when we would convert a Base attestation to Electra / how to handle mismatched attestations here - (IndexedAttestation::Base(existing_att), IndexedAttestation::Base(new_att)) => { - Some(AttesterSlashing::Base(AttesterSlashingBase { - attestation_1: existing_att, - attestation_2: new_att.clone(), - })) - } - ( - IndexedAttestation::Electra(existing_att), - IndexedAttestation::Electra(new_att), - ) => Some(AttesterSlashing::Electra(AttesterSlashingElectra { + DoubleVote(existing) | SurroundedByExisting(existing) => match *existing { + IndexedAttestation::Base(existing_att) => { + Some(AttesterSlashing::Base(AttesterSlashingBase { attestation_1: existing_att, - attestation_2: new_att.clone(), - })), - _ => panic!("attestations must be of the same type"), + attestation_2: new_attestation + .as_base() + .map_err(|e| format!("{e:?}"))? + .clone(), + })) } - } - // TODO(electra): fix this once we superstruct IndexedAttestation (return the correct type) - SurroundsExisting(existing) => match (*existing, new_attestation) { - (IndexedAttestation::Base(existing_att), IndexedAttestation::Base(new_att)) => { - Some(AttesterSlashing::Base(AttesterSlashingBase { - attestation_1: new_att.clone(), - attestation_2: existing_att, + IndexedAttestation::Electra(existing_att) => { + Some(AttesterSlashing::Electra(AttesterSlashingElectra { + attestation_1: existing_att, + // A double vote should never convert, a surround vote where the surrounding + // vote is electra may convert. + attestation_2: new_attestation + .clone() + .to_electra() + .map_err(|e| format!("{e:?}"))?, })) } - ( - IndexedAttestation::Electra(existing_att), - IndexedAttestation::Electra(new_att), - ) => Some(AttesterSlashing::Electra(AttesterSlashingElectra { - attestation_1: new_att.clone(), - attestation_2: existing_att, - })), - _ => panic!("attestations must be of the same type"), }, - /* - Some(AttesterSlashing::Base(AttesterSlashingBase { - attestation_1: new_attestation.clone(), - attestation_2: *existing, - })), - */ - } + SurroundsExisting(existing) => { + match new_attestation { + IndexedAttestation::Base(new_attestation) => { + Some(AttesterSlashing::Base(AttesterSlashingBase { + attestation_1: existing + .as_base() + .map_err(|e| format!("{e:?}"))? + .clone(), + attestation_2: new_attestation.clone(), + })) + } + IndexedAttestation::Electra(new_attestation) => { + Some(AttesterSlashing::Electra(AttesterSlashingElectra { + attestation_1: existing.to_electra().map_err(|e| format!("{e:?}"))?, + // A double vote should never convert, a surround vote where the surrounding + // vote is electra may convert. + attestation_2: new_attestation.clone(), + })) + } + } + } + }) } } diff --git a/slasher/src/slasher.rs b/slasher/src/slasher.rs index fc8e8453c89..ef6dcce9b1b 100644 --- a/slasher/src/slasher.rs +++ b/slasher/src/slasher.rs @@ -247,6 +247,7 @@ impl Slasher { batch, current_epoch, &self.config, + &self.log, ) { Ok(slashings) => { if !slashings.is_empty() { @@ -294,14 +295,25 @@ impl Slasher { indexed_attestation_id, )?; - if let Some(slashing) = slashing_status.into_slashing(attestation) { - debug!( - self.log, - "Found double-vote slashing"; - "validator_index" => validator_index, - "epoch" => slashing.attestation_1().data().target.epoch, - ); - slashings.insert(slashing); + match slashing_status.into_slashing(attestation) { + Ok(Some(slashing)) => { + debug!( + self.log, + "Found double-vote slashing"; + "validator_index" => validator_index, + "epoch" => slashing.attestation_1().data().target.epoch, + ); + slashings.insert(slashing); + } + Err(e) => { + error!( + self.log, + "Invalid slashing conversion"; + "validator_index" => validator_index, + "error" => e + ); + } + Ok(None) => {} } } From f9c50bca076cbadd449d3cbb8db3b0c0ec3a1922 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Thu, 16 May 2024 12:55:03 +0300 Subject: [PATCH 115/227] Fix Bug In Block Processing with 0x02 Credentials --- .../state_processing/src/per_block_processing.rs | 4 ++-- consensus/types/src/validator.rs | 12 ------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 0f67dd95f85..05621d188be 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -566,7 +566,7 @@ pub fn get_expected_withdrawals( index: withdrawal_index, validator_index, address: validator - .get_eth1_withdrawal_address(spec) + .get_execution_withdrawal_address(spec) .ok_or(BlockProcessingError::WithdrawalCredentialsInvalid)?, amount: balance, }); @@ -576,7 +576,7 @@ pub fn get_expected_withdrawals( index: withdrawal_index, validator_index, address: validator - .get_eth1_withdrawal_address(spec) + .get_execution_withdrawal_address(spec) .ok_or(BlockProcessingError::WithdrawalCredentialsInvalid)?, amount: balance.safe_sub( validator diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index 86243083f06..d4d166bdcb3 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -130,18 +130,6 @@ impl Validator { is_compounding_withdrawal_credential(self.withdrawal_credentials, spec) } - /// Get the eth1 withdrawal address if this validator has one initialized. - pub fn get_eth1_withdrawal_address(&self, spec: &ChainSpec) -> Option
{ - self.has_eth1_withdrawal_credential(spec) - .then(|| { - self.withdrawal_credentials - .as_bytes() - .get(12..) - .map(Address::from_slice) - }) - .flatten() - } - /// Get the execution withdrawal address if this validator has one initialized. pub fn get_execution_withdrawal_address(&self, spec: &ChainSpec) -> Option
{ self.has_execution_withdrawal_credential(spec) From 82858bc04e2f40df489f8b35fc36b1dc721470f2 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Mon, 13 May 2024 10:18:17 +0300 Subject: [PATCH 116/227] Send unagg attestation based on fork --- validator_client/src/attestation_service.rs | 32 +++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 5b7f31867bb..f6cc4629634 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -18,6 +18,7 @@ use types::{ attestation::AttestationBase, AggregateSignature, Attestation, AttestationData, BitList, ChainSpec, CommitteeIndex, EthSpec, Slot, }; +use types::{AttestationElectra, BitVector, ForkName}; /// Builds an `AttestationService`. pub struct AttestationServiceBuilder { @@ -378,11 +379,32 @@ impl AttestationService { return None; } - let mut attestation = Attestation::Base(AttestationBase { - aggregation_bits: BitList::with_capacity(duty.committee_length as usize).unwrap(), - data: attestation_data.clone(), - signature: AggregateSignature::infinity(), - }); + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); + + let mut attestation = if fork_name >= ForkName::Electra { + let mut committee_bits: BitVector = BitVector::default(); + committee_bits + .set(duty.committee_index as usize, true) + .unwrap(); + Attestation::Electra(AttestationElectra { + aggregation_bits: BitList::with_capacity(duty.committee_length as usize) + .unwrap(), + data: attestation_data.clone(), + committee_bits, + signature: AggregateSignature::infinity(), + }) + } else { + Attestation::Base(AttestationBase { + aggregation_bits: BitList::with_capacity(duty.committee_length as usize) + .unwrap(), + data: attestation_data.clone(), + signature: AggregateSignature::infinity(), + }) + }; match self .validator_store From 154b7a7b8a9a12cbe19cd55288acfa111f36486b Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Wed, 15 May 2024 15:50:38 +0300 Subject: [PATCH 117/227] Publish all aggregates --- validator_client/src/attestation_service.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index f6cc4629634..c172399fd3f 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -363,9 +363,15 @@ impl AttestationService { let duty = &duty_and_proof.duty; let attestation_data = attestation_data_ref; + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); + // Ensure that the attestation matches the duties. #[allow(clippy::suspicious_operation_groupings)] - if duty.slot != attestation_data.slot || duty.committee_index != attestation_data.index + if duty.slot != attestation_data.slot || (fork_name < ForkName::Electra && duty.committee_index != attestation_data.index) { crit!( log, @@ -379,12 +385,6 @@ impl AttestationService { return None; } - let fork_name = self - .context - .eth2_config - .spec - .fork_name_at_slot::(attestation_data.slot); - let mut attestation = if fork_name >= ForkName::Electra { let mut committee_bits: BitVector = BitVector::default(); committee_bits From bb734afa1d781382248c67b06b5b6c8f23c1ddbf Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Wed, 15 May 2024 17:05:40 +0300 Subject: [PATCH 118/227] just one more check bro plz.. --- validator_client/src/attestation_service.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index c172399fd3f..4213edc4392 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -371,7 +371,8 @@ impl AttestationService { // Ensure that the attestation matches the duties. #[allow(clippy::suspicious_operation_groupings)] - if duty.slot != attestation_data.slot || (fork_name < ForkName::Electra && duty.committee_index != attestation_data.index) + if duty.slot != attestation_data.slot + || (fork_name < ForkName::Electra && duty.committee_index != attestation_data.index) { crit!( log, @@ -553,6 +554,12 @@ impl AttestationService { .await .map_err(|e| e.to_string())?; + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); + // Create futures to produce the signed aggregated attestations. let signing_futures = validator_duties.iter().map(|duty_and_proof| async move { let duty = &duty_and_proof.duty; @@ -561,7 +568,9 @@ impl AttestationService { let slot = attestation_data.slot; let committee_index = attestation_data.index; - if duty.slot != slot || duty.committee_index != committee_index { + if duty.slot != slot + || (fork_name < ForkName::Electra && duty.committee_index != committee_index) + { crit!(log, "Inconsistent validator duties during signing"); return None; } From 7192f003880ad70eb132a212f7208a8806f290ad Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 28 May 2024 12:43:38 +0530 Subject: [PATCH 119/227] Rename deploy block electra (#5853) * Rename deploy_block.txt to deposit_contract_block.txt * fmt * Change for prater too --- common/eth2_config/src/lib.rs | 6 +++++- .../chiado/{deploy_block.txt => deposit_contract_block.txt} | 0 .../gnosis/{deploy_block.txt => deposit_contract_block.txt} | 0 .../{deploy_block.txt => deposit_contract_block.txt} | 0 .../{deploy_block.txt => deposit_contract_block.txt} | 0 .../prater/{deploy_block.txt => deposit_contract_block.txt} | 0 .../{deploy_block.txt => deposit_contract_block.txt} | 0 common/eth2_network_config/src/lib.rs | 2 +- 8 files changed, 6 insertions(+), 2 deletions(-) rename common/eth2_network_config/built_in_network_configs/chiado/{deploy_block.txt => deposit_contract_block.txt} (100%) rename common/eth2_network_config/built_in_network_configs/gnosis/{deploy_block.txt => deposit_contract_block.txt} (100%) rename common/eth2_network_config/built_in_network_configs/holesky/{deploy_block.txt => deposit_contract_block.txt} (100%) rename common/eth2_network_config/built_in_network_configs/mainnet/{deploy_block.txt => deposit_contract_block.txt} (100%) rename common/eth2_network_config/built_in_network_configs/prater/{deploy_block.txt => deposit_contract_block.txt} (100%) rename common/eth2_network_config/built_in_network_configs/sepolia/{deploy_block.txt => deposit_contract_block.txt} (100%) diff --git a/common/eth2_config/src/lib.rs b/common/eth2_config/src/lib.rs index bf707c4d17b..e41ba0ad431 100644 --- a/common/eth2_config/src/lib.rs +++ b/common/eth2_config/src/lib.rs @@ -192,7 +192,11 @@ macro_rules! define_net { config_dir: ETH2_NET_DIR.config_dir, genesis_state_source: ETH2_NET_DIR.genesis_state_source, config: $this_crate::$include_file!($this_crate, "../", "config.yaml"), - deploy_block: $this_crate::$include_file!($this_crate, "../", "deploy_block.txt"), + deploy_block: $this_crate::$include_file!( + $this_crate, + "../", + "deposit_contract_block.txt" + ), boot_enr: $this_crate::$include_file!($this_crate, "../", "boot_enr.yaml"), genesis_state_bytes: $this_crate::$include_file!($this_crate, "../", "genesis.ssz"), } diff --git a/common/eth2_network_config/built_in_network_configs/chiado/deploy_block.txt b/common/eth2_network_config/built_in_network_configs/chiado/deposit_contract_block.txt similarity index 100% rename from common/eth2_network_config/built_in_network_configs/chiado/deploy_block.txt rename to common/eth2_network_config/built_in_network_configs/chiado/deposit_contract_block.txt diff --git a/common/eth2_network_config/built_in_network_configs/gnosis/deploy_block.txt b/common/eth2_network_config/built_in_network_configs/gnosis/deposit_contract_block.txt similarity index 100% rename from common/eth2_network_config/built_in_network_configs/gnosis/deploy_block.txt rename to common/eth2_network_config/built_in_network_configs/gnosis/deposit_contract_block.txt diff --git a/common/eth2_network_config/built_in_network_configs/holesky/deploy_block.txt b/common/eth2_network_config/built_in_network_configs/holesky/deposit_contract_block.txt similarity index 100% rename from common/eth2_network_config/built_in_network_configs/holesky/deploy_block.txt rename to common/eth2_network_config/built_in_network_configs/holesky/deposit_contract_block.txt diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/deploy_block.txt b/common/eth2_network_config/built_in_network_configs/mainnet/deposit_contract_block.txt similarity index 100% rename from common/eth2_network_config/built_in_network_configs/mainnet/deploy_block.txt rename to common/eth2_network_config/built_in_network_configs/mainnet/deposit_contract_block.txt diff --git a/common/eth2_network_config/built_in_network_configs/prater/deploy_block.txt b/common/eth2_network_config/built_in_network_configs/prater/deposit_contract_block.txt similarity index 100% rename from common/eth2_network_config/built_in_network_configs/prater/deploy_block.txt rename to common/eth2_network_config/built_in_network_configs/prater/deposit_contract_block.txt diff --git a/common/eth2_network_config/built_in_network_configs/sepolia/deploy_block.txt b/common/eth2_network_config/built_in_network_configs/sepolia/deposit_contract_block.txt similarity index 100% rename from common/eth2_network_config/built_in_network_configs/sepolia/deploy_block.txt rename to common/eth2_network_config/built_in_network_configs/sepolia/deposit_contract_block.txt diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 1ead9a6bde8..95cb2acd40e 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -29,7 +29,7 @@ use url::Url; pub use eth2_config::GenesisStateSource; -pub const DEPLOY_BLOCK_FILE: &str = "deploy_block.txt"; +pub const DEPLOY_BLOCK_FILE: &str = "deposit_contract_block.txt"; pub const BOOT_ENR_FILE: &str = "boot_enr.yaml"; pub const GENESIS_STATE_FILE: &str = "genesis.ssz"; pub const BASE_CONFIG_FILE: &str = "config.yaml"; From aed25c49e318ab39fb1c14e11d74a07afde662a9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 28 May 2024 14:06:33 +0200 Subject: [PATCH 120/227] fix: serde rename camle case for execution payload body (#5846) --- beacon_node/execution_layer/src/engine_api/json_structures.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 034483eb73b..f0d79303510 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -720,6 +720,7 @@ impl From for JsonForkchoiceUpdatedV1Response { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(bound = "E: EthSpec")] +#[serde(rename_all = "camelCase")] pub struct JsonExecutionPayloadBodyV1 { #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, From 75432e113571b5684ded88137d8bab226ce02e47 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Thu, 30 May 2024 17:34:14 +0200 Subject: [PATCH 121/227] Electra attestation changes rm decode impl (#5856) * Remove Crappy Decode impl for Attestation * Remove Inefficient Attestation Decode impl * Implement Schema Upgrade / Downgrade * Update beacon_node/beacon_chain/src/schema_change/migration_schema_v20.rs Co-authored-by: Michael Sproul --------- Co-authored-by: Michael Sproul --- beacon_node/beacon_chain/src/schema_change.rs | 9 ++ .../src/schema_change/migration_schema_v20.rs | 103 ++++++++++++ .../lighthouse_network/src/types/pubsub.rs | 36 ++++- beacon_node/operation_pool/src/lib.rs | 3 +- beacon_node/operation_pool/src/persistence.rs | 152 +++++++++--------- beacon_node/store/src/metadata.rs | 2 +- .../state_processing/src/verify_operation.rs | 47 +++++- consensus/types/src/attestation.rs | 80 ++++++--- testing/ef_tests/src/cases/fork_choice.rs | 22 ++- testing/ef_tests/src/cases/operations.rs | 8 +- 10 files changed, 343 insertions(+), 119 deletions(-) create mode 100644 beacon_node/beacon_chain/src/schema_change/migration_schema_v20.rs diff --git a/beacon_node/beacon_chain/src/schema_change.rs b/beacon_node/beacon_chain/src/schema_change.rs index 63eb72c43ab..06d189a8c01 100644 --- a/beacon_node/beacon_chain/src/schema_change.rs +++ b/beacon_node/beacon_chain/src/schema_change.rs @@ -2,6 +2,7 @@ mod migration_schema_v17; mod migration_schema_v18; mod migration_schema_v19; +mod migration_schema_v20; use crate::beacon_chain::BeaconChainTypes; use crate::types::ChainSpec; @@ -78,6 +79,14 @@ pub fn migrate_schema( let ops = migration_schema_v19::downgrade_from_v19::(db.clone(), log)?; db.store_schema_version_atomically(to, ops) } + (SchemaVersion(19), SchemaVersion(20)) => { + let ops = migration_schema_v20::upgrade_to_v20::(db.clone(), log)?; + db.store_schema_version_atomically(to, ops) + } + (SchemaVersion(20), SchemaVersion(19)) => { + let ops = migration_schema_v20::downgrade_from_v20::(db.clone(), log)?; + db.store_schema_version_atomically(to, ops) + } // Anything else is an error. (_, _) => Err(HotColdDBError::UnsupportedSchemaVersion { target_version: to, diff --git a/beacon_node/beacon_chain/src/schema_change/migration_schema_v20.rs b/beacon_node/beacon_chain/src/schema_change/migration_schema_v20.rs new file mode 100644 index 00000000000..737fcd0a935 --- /dev/null +++ b/beacon_node/beacon_chain/src/schema_change/migration_schema_v20.rs @@ -0,0 +1,103 @@ +use crate::beacon_chain::{BeaconChainTypes, OP_POOL_DB_KEY}; +use operation_pool::{ + PersistedOperationPool, PersistedOperationPoolV15, PersistedOperationPoolV20, +}; +use slog::{debug, info, Logger}; +use std::sync::Arc; +use store::{Error, HotColdDB, KeyValueStoreOp, StoreItem}; +use types::Attestation; + +pub fn upgrade_to_v20( + db: Arc>, + log: Logger, +) -> Result, Error> { + // Load a V15 op pool and transform it to V20. + let Some(PersistedOperationPoolV15:: { + attestations_v15, + sync_contributions, + attester_slashings_v15, + proposer_slashings, + voluntary_exits, + bls_to_execution_changes, + capella_bls_change_broadcast_indices, + }) = db.get_item(&OP_POOL_DB_KEY)? + else { + debug!(log, "Nothing to do, no operation pool stored"); + return Ok(vec![]); + }; + + let attestations = attestations_v15 + .into_iter() + .map(|(attestation, indices)| (Attestation::Base(attestation).into(), indices)) + .collect(); + + let attester_slashings = attester_slashings_v15 + .into_iter() + .map(|slashing| slashing.into()) + .collect(); + + let v20 = PersistedOperationPool::V20(PersistedOperationPoolV20 { + attestations, + sync_contributions, + attester_slashings, + proposer_slashings, + voluntary_exits, + bls_to_execution_changes, + capella_bls_change_broadcast_indices, + }); + Ok(vec![v20.as_kv_store_op(OP_POOL_DB_KEY)]) +} + +pub fn downgrade_from_v20( + db: Arc>, + log: Logger, +) -> Result, Error> { + // Load a V20 op pool and transform it to V15. + let Some(PersistedOperationPoolV20:: { + attestations, + sync_contributions, + attester_slashings, + proposer_slashings, + voluntary_exits, + bls_to_execution_changes, + capella_bls_change_broadcast_indices, + }) = db.get_item(&OP_POOL_DB_KEY)? + else { + debug!(log, "Nothing to do, no operation pool stored"); + return Ok(vec![]); + }; + + let attestations_v15 = attestations + .into_iter() + .filter_map(|(attestation, indices)| { + if let Attestation::Base(attestation) = attestation.into() { + Some((attestation, indices)) + } else { + info!(log, "Dropping attestation during downgrade"; "reason" => "not a base attestation"); + None + } + }) + .collect(); + + let attester_slashings_v15 = attester_slashings + .into_iter() + .filter_map(|slashing| match slashing.try_into() { + Ok(slashing) => Some(slashing), + Err(_) => { + info!(log, "Dropping attester slashing during downgrade"; "reason" => "not a base attester slashing"); + None + } + }) + .collect(); + + let v15 = PersistedOperationPool::V15(PersistedOperationPoolV15 { + attestations_v15, + sync_contributions, + attester_slashings_v15, + proposer_slashings, + voluntary_exits, + bls_to_execution_changes, + capella_bls_change_broadcast_indices, + }); + Ok(vec![v15.as_kv_store_op(OP_POOL_DB_KEY)]) +} diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 8945c23a549..346c177e582 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -7,13 +7,14 @@ use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; use types::{ - Attestation, AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, - EthSpec, ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, - ProposerSlashing, SignedAggregateAndProof, SignedAggregateAndProofBase, - SignedAggregateAndProofElectra, SignedBeaconBlock, SignedBeaconBlockAltair, - SignedBeaconBlockBase, SignedBeaconBlockBellatrix, SignedBeaconBlockCapella, - SignedBeaconBlockDeneb, SignedBeaconBlockElectra, SignedBlsToExecutionChange, - SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, + Attestation, AttestationBase, AttestationElectra, AttesterSlashing, AttesterSlashingBase, + AttesterSlashingElectra, BlobSidecar, EthSpec, ForkContext, ForkName, + LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, + SignedAggregateAndProof, SignedAggregateAndProofBase, SignedAggregateAndProofElectra, + SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, + SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, + SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, + SyncCommitteeMessage, SyncSubnetId, }; #[derive(Debug, Clone, PartialEq)] @@ -182,7 +183,26 @@ impl PubsubMessage { } GossipKind::Attestation(subnet_id) => { let attestation = - Attestation::from_ssz_bytes(data).map_err(|e| format!("{:?}", e))?; + match fork_context.from_context_bytes(gossip_topic.fork_digest) { + Some(ForkName::Base) + | Some(ForkName::Altair) + | Some(ForkName::Bellatrix) + | Some(ForkName::Capella) + | Some(ForkName::Deneb) => Attestation::Base( + AttestationBase::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ), + Some(ForkName::Electra) => Attestation::Electra( + AttestationElectra::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ), + None => { + return Err(format!( + "Unknown gossipsub fork digest: {:?}", + gossip_topic.fork_digest + )) + } + }; Ok(PubsubMessage::Attestation(Box::new(( *subnet_id, attestation, diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index daddbf76652..be9f4902892 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -14,8 +14,7 @@ pub use attestation::{earliest_attestation_validators, AttMaxCover}; pub use attestation_storage::{AttestationRef, SplitAttestation}; pub use max_cover::MaxCover; pub use persistence::{ - PersistedOperationPool, PersistedOperationPoolV12, PersistedOperationPoolV14, - PersistedOperationPoolV15, PersistedOperationPoolV5, + PersistedOperationPool, PersistedOperationPoolV15, PersistedOperationPoolV20, }; pub use reward_cache::RewardCache; use state_processing::epoch_cache::is_epoch_cache_initialized; diff --git a/beacon_node/operation_pool/src/persistence.rs b/beacon_node/operation_pool/src/persistence.rs index 6fd8a6cc3cc..99cb1aafbc5 100644 --- a/beacon_node/operation_pool/src/persistence.rs +++ b/beacon_node/operation_pool/src/persistence.rs @@ -1,4 +1,3 @@ -use crate::attestation_id::AttestationId; use crate::attestation_storage::AttestationMap; use crate::bls_to_execution_changes::{BlsToExecutionChanges, ReceivedPreCapella}; use crate::sync_aggregate_id::SyncAggregateId; @@ -12,6 +11,7 @@ use state_processing::SigVerifiedOp; use std::collections::HashSet; use std::mem; use store::{DBColumn, Error as StoreError, StoreItem}; +use types::attestation::AttestationOnDisk; use types::*; type PersistedSyncContributions = Vec<(SyncAggregateId, Vec>)>; @@ -21,7 +21,7 @@ type PersistedSyncContributions = Vec<(SyncAggregateId, Vec = Vec<(SyncAggregateId, Vec { - /// [DEPRECATED] Mapping from attestation ID to attestation mappings. - #[superstruct(only(V5))] - pub attestations_v5: Vec<(AttestationId, Vec>)>, + #[superstruct(only(V15))] + pub attestations_v15: Vec<(AttestationBase, Vec)>, /// Attestations and their attesting indices. - #[superstruct(only(V12, V14, V15))] - pub attestations: Vec<(Attestation, Vec)>, + #[superstruct(only(V20))] + pub attestations: Vec<(AttestationOnDisk, Vec)>, /// Mapping from sync contribution ID to sync contributions and aggregate. pub sync_contributions: PersistedSyncContributions, - /// TODO(electra): we've made a DB change here!!! + #[superstruct(only(V15))] + pub attester_slashings_v15: Vec, E>>, /// Attester slashings. - #[superstruct(only(V12, V14, V15))] + #[superstruct(only(V20))] pub attester_slashings: Vec, E>>, - /// [DEPRECATED] Proposer slashings. - #[superstruct(only(V5))] - pub proposer_slashings_v5: Vec, /// Proposer slashings with fork information. - #[superstruct(only(V12, V14, V15))] pub proposer_slashings: Vec>, - /// [DEPRECATED] Voluntary exits. - #[superstruct(only(V5))] - pub voluntary_exits_v5: Vec, /// Voluntary exits with fork information. - #[superstruct(only(V12, V14, V15))] pub voluntary_exits: Vec>, /// BLS to Execution Changes - #[superstruct(only(V14, V15))] pub bls_to_execution_changes: Vec>, /// Validator indices with BLS to Execution Changes to be broadcast at the /// Capella fork. - #[superstruct(only(V15))] pub capella_bls_change_broadcast_indices: Vec, } @@ -73,7 +63,7 @@ impl PersistedOperationPool { .iter() .map(|att| { ( - att.clone_as_attestation(), + AttestationOnDisk::from(att.clone_as_attestation()), att.indexed.attesting_indices().clone(), ) }) @@ -121,7 +111,7 @@ impl PersistedOperationPool { .copied() .collect(); - PersistedOperationPool::V15(PersistedOperationPoolV15 { + PersistedOperationPool::V20(PersistedOperationPoolV20 { attestations, sync_contributions, attester_slashings, @@ -134,56 +124,86 @@ impl PersistedOperationPool { /// Reconstruct an `OperationPool`. pub fn into_operation_pool(mut self) -> Result, OpPoolError> { - let attester_slashings = RwLock::new(self.attester_slashings()?.iter().cloned().collect()); + let attester_slashings = match &self { + PersistedOperationPool::V15(pool_v15) => RwLock::new( + pool_v15 + .attester_slashings_v15 + .iter() + .map(|slashing| slashing.clone().into()) + .collect(), + ), + PersistedOperationPool::V20(pool_v20) => { + RwLock::new(pool_v20.attester_slashings.iter().cloned().collect()) + } + }; + let proposer_slashings = RwLock::new( - self.proposer_slashings()? + self.proposer_slashings() .iter() .cloned() .map(|slashing| (slashing.as_inner().proposer_index(), slashing)) .collect(), ); let voluntary_exits = RwLock::new( - self.voluntary_exits()? + self.voluntary_exits() .iter() .cloned() .map(|exit| (exit.as_inner().message.validator_index, exit)) .collect(), ); let sync_contributions = RwLock::new(self.sync_contributions().iter().cloned().collect()); - let attestations = match self { - PersistedOperationPool::V5(_) | PersistedOperationPool::V12(_) => { - return Err(OpPoolError::IncorrectOpPoolVariant) + let attestations = match &self { + PersistedOperationPool::V15(pool_v15) => { + let mut map = AttestationMap::default(); + for (att, attesting_indices) in + pool_v15 + .attestations_v15 + .iter() + .map(|(att, attesting_indices)| { + (Attestation::Base(att.clone()), attesting_indices.clone()) + }) + { + map.insert(att, attesting_indices); + } + RwLock::new(map) } - PersistedOperationPool::V14(_) | PersistedOperationPool::V15(_) => { + PersistedOperationPool::V20(pool_v20) => { let mut map = AttestationMap::default(); - for (att, attesting_indices) in self.attestations()?.clone() { + for (att, attesting_indices) in + pool_v20 + .attestations + .iter() + .map(|(att, attesting_indices)| { + ( + AttestationRef::from(att.to_ref()).clone_as_attestation(), + attesting_indices.clone(), + ) + }) + { map.insert(att, attesting_indices); } RwLock::new(map) } }; - let mut bls_to_execution_changes = BlsToExecutionChanges::default(); - if let Ok(persisted_changes) = self.bls_to_execution_changes_mut() { - let persisted_changes = mem::take(persisted_changes); - let broadcast_indices = - if let Ok(indices) = self.capella_bls_change_broadcast_indices_mut() { - mem::take(indices).into_iter().collect() - } else { - HashSet::new() - }; + let mut bls_to_execution_changes = BlsToExecutionChanges::default(); + let persisted_changes = mem::take(self.bls_to_execution_changes_mut()); + let broadcast_indices: HashSet<_> = + mem::take(self.capella_bls_change_broadcast_indices_mut()) + .into_iter() + .collect(); - for bls_to_execution_change in persisted_changes { - let received_pre_capella = if broadcast_indices - .contains(&bls_to_execution_change.as_inner().message.validator_index) - { - ReceivedPreCapella::Yes - } else { - ReceivedPreCapella::No - }; - bls_to_execution_changes.insert(bls_to_execution_change, received_pre_capella); - } + for bls_to_execution_change in persisted_changes { + let received_pre_capella = if broadcast_indices + .contains(&bls_to_execution_change.as_inner().message.validator_index) + { + ReceivedPreCapella::Yes + } else { + ReceivedPreCapella::No + }; + bls_to_execution_changes.insert(bls_to_execution_change, received_pre_capella); } + let op_pool = OperationPool { attestations, sync_contributions, @@ -198,35 +218,7 @@ impl PersistedOperationPool { } } -impl StoreItem for PersistedOperationPoolV5 { - fn db_column() -> DBColumn { - DBColumn::OpPool - } - - fn as_store_bytes(&self) -> Vec { - self.as_ssz_bytes() - } - - fn from_store_bytes(bytes: &[u8]) -> Result { - PersistedOperationPoolV5::from_ssz_bytes(bytes).map_err(Into::into) - } -} - -impl StoreItem for PersistedOperationPoolV12 { - fn db_column() -> DBColumn { - DBColumn::OpPool - } - - fn as_store_bytes(&self) -> Vec { - self.as_ssz_bytes() - } - - fn from_store_bytes(bytes: &[u8]) -> Result { - PersistedOperationPoolV12::from_ssz_bytes(bytes).map_err(Into::into) - } -} - -impl StoreItem for PersistedOperationPoolV14 { +impl StoreItem for PersistedOperationPoolV15 { fn db_column() -> DBColumn { DBColumn::OpPool } @@ -236,11 +228,11 @@ impl StoreItem for PersistedOperationPoolV14 { } fn from_store_bytes(bytes: &[u8]) -> Result { - PersistedOperationPoolV14::from_ssz_bytes(bytes).map_err(Into::into) + PersistedOperationPoolV15::from_ssz_bytes(bytes).map_err(Into::into) } } -impl StoreItem for PersistedOperationPoolV15 { +impl StoreItem for PersistedOperationPoolV20 { fn db_column() -> DBColumn { DBColumn::OpPool } @@ -250,7 +242,7 @@ impl StoreItem for PersistedOperationPoolV15 { } fn from_store_bytes(bytes: &[u8]) -> Result { - PersistedOperationPoolV15::from_ssz_bytes(bytes).map_err(Into::into) + PersistedOperationPoolV20::from_ssz_bytes(bytes).map_err(Into::into) } } diff --git a/beacon_node/store/src/metadata.rs b/beacon_node/store/src/metadata.rs index 1675051bd80..116926ad3f1 100644 --- a/beacon_node/store/src/metadata.rs +++ b/beacon_node/store/src/metadata.rs @@ -4,7 +4,7 @@ use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use types::{Checkpoint, Hash256, Slot}; -pub const CURRENT_SCHEMA_VERSION: SchemaVersion = SchemaVersion(19); +pub const CURRENT_SCHEMA_VERSION: SchemaVersion = SchemaVersion(20); // All the keys that get stored under the `BeaconMeta` column. // diff --git a/consensus/state_processing/src/verify_operation.rs b/consensus/state_processing/src/verify_operation.rs index c12467fbb57..f218b806d20 100644 --- a/consensus/state_processing/src/verify_operation.rs +++ b/consensus/state_processing/src/verify_operation.rs @@ -12,7 +12,9 @@ use smallvec::{smallvec, SmallVec}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::marker::PhantomData; -use types::{AttesterSlashing, AttesterSlashingOnDisk, AttesterSlashingRefOnDisk}; +use types::{ + AttesterSlashing, AttesterSlashingBase, AttesterSlashingOnDisk, AttesterSlashingRefOnDisk, +}; use types::{ BeaconState, ChainSpec, Epoch, EthSpec, Fork, ForkVersion, ProposerSlashing, SignedBlsToExecutionChange, SignedVoluntaryExit, @@ -366,6 +368,49 @@ impl TransformPersist for AttesterSlashing { } } +// TODO: Remove this once we no longer support DB schema version 17 +impl TransformPersist for types::AttesterSlashingBase { + type Persistable = Self; + type PersistableRef<'a> = &'a Self; + + fn as_persistable_ref(&self) -> Self::PersistableRef<'_> { + self + } + + fn from_persistable(persistable: Self::Persistable) -> Self { + persistable + } +} +// TODO: Remove this once we no longer support DB schema version 17 +impl From, E>> + for SigVerifiedOp, E> +{ + fn from(base: SigVerifiedOp, E>) -> Self { + SigVerifiedOp { + op: AttesterSlashing::Base(base.op), + verified_against: base.verified_against, + _phantom: PhantomData, + } + } +} +// TODO: Remove this once we no longer support DB schema version 17 +impl TryFrom, E>> + for SigVerifiedOp, E> +{ + type Error = String; + + fn try_from(slashing: SigVerifiedOp, E>) -> Result { + match slashing.op { + AttesterSlashing::Base(base) => Ok(SigVerifiedOp { + op: base, + verified_against: slashing.verified_against, + _phantom: PhantomData, + }), + AttesterSlashing::Electra(_) => Err("non-base attester slashing".to_string()), + } + } +} + impl TransformPersist for ProposerSlashing { type Persistable = Self; type PersistableRef<'a> = &'a Self; diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 8c8a81b90f2..a2c5140395b 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -4,7 +4,6 @@ use derivative::Derivative; use rand::RngCore; use safe_arith::ArithError; use serde::{Deserialize, Serialize}; -use ssz::Decode; use ssz_derive::{Decode, Encode}; use ssz_types::BitVector; use std::hash::{Hash, Hasher}; @@ -72,26 +71,6 @@ pub struct Attestation { pub signature: AggregateSignature, } -impl Decode for Attestation { - fn is_ssz_fixed_len() -> bool { - false - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - if let Ok(result) = AttestationBase::from_ssz_bytes(bytes) { - return Ok(Attestation::Base(result)); - } - - if let Ok(result) = AttestationElectra::from_ssz_bytes(bytes) { - return Ok(Attestation::Electra(result)); - } - - Err(ssz::DecodeError::BytesInvalid(String::from( - "bytes not valid for any fork variant", - ))) - } -} - // TODO(electra): think about how to handle fork variants here impl TestRandom for Attestation { fn random_for_test(rng: &mut impl RngCore) -> Self { @@ -417,6 +396,65 @@ impl<'a, E: EthSpec> SlotData for AttestationRef<'a, E> { } } +#[derive(Debug, Clone, Encode, Decode, PartialEq)] +#[ssz(enum_behaviour = "union")] +pub enum AttestationOnDisk { + Base(AttestationBase), + Electra(AttestationElectra), +} + +impl AttestationOnDisk { + pub fn to_ref(&self) -> AttestationRefOnDisk { + match self { + AttestationOnDisk::Base(att) => AttestationRefOnDisk::Base(att), + AttestationOnDisk::Electra(att) => AttestationRefOnDisk::Electra(att), + } + } +} + +#[derive(Debug, Clone, Encode)] +#[ssz(enum_behaviour = "union")] +pub enum AttestationRefOnDisk<'a, E: EthSpec> { + Base(&'a AttestationBase), + Electra(&'a AttestationElectra), +} + +impl From> for AttestationOnDisk { + fn from(attestation: Attestation) -> Self { + match attestation { + Attestation::Base(attestation) => Self::Base(attestation), + Attestation::Electra(attestation) => Self::Electra(attestation), + } + } +} + +impl From> for Attestation { + fn from(attestation: AttestationOnDisk) -> Self { + match attestation { + AttestationOnDisk::Base(attestation) => Self::Base(attestation), + AttestationOnDisk::Electra(attestation) => Self::Electra(attestation), + } + } +} + +impl<'a, E: EthSpec> From> for AttestationRefOnDisk<'a, E> { + fn from(attestation: AttestationRef<'a, E>) -> Self { + match attestation { + AttestationRef::Base(attestation) => Self::Base(attestation), + AttestationRef::Electra(attestation) => Self::Electra(attestation), + } + } +} + +impl<'a, E: EthSpec> From> for AttestationRef<'a, E> { + fn from(attestation: AttestationRefOnDisk<'a, E>) -> Self { + match attestation { + AttestationRefOnDisk::Base(attestation) => Self::Base(attestation), + AttestationRefOnDisk::Electra(attestation) => Self::Electra(attestation), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 2e39294582a..b3b20d6efe1 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -180,10 +180,24 @@ impl LoadCase for ForkChoiceTest { valid, }) } - Step::Attestation { attestation } => { - ssz_decode_file(&path.join(format!("{}.ssz_snappy", attestation))) - .map(|attestation| Step::Attestation { attestation }) - } + Step::Attestation { attestation } => match fork_name { + ForkName::Base + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => ssz_decode_file( + &path.join(format!("{}.ssz_snappy", attestation)), + ) + .map(|attestation| Step::Attestation { + attestation: Attestation::Base(attestation), + }), + ForkName::Electra => ssz_decode_file( + &path.join(format!("{}.ssz_snappy", attestation)), + ) + .map(|attestation| Step::Attestation { + attestation: Attestation::Electra(attestation), + }), + }, Step::AttesterSlashing { attester_slashing } => match fork_name { ForkName::Base | ForkName::Altair diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index b922668c0a0..d87770a4df0 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -78,8 +78,12 @@ impl Operation for Attestation { "attestation".into() } - fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { - ssz_decode_file(path) + fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { + if fork_name < ForkName::Electra { + Ok(Self::Base(ssz_decode_file(path)?)) + } else { + Ok(Self::Electra(ssz_decode_file(path)?)) + } } fn apply_to( From e3409982416da34e8c4e40f9b446441d70e6e485 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Thu, 30 May 2024 17:51:34 +0200 Subject: [PATCH 122/227] Fix failing attestation tests and misc electra attestation cleanup (#5810) * - get attestation related beacon chain tests to pass - observed attestations are now keyed off of data + committee index - rename op pool attestationref to compactattestationref - remove unwraps in agg pool and use options instead - cherry pick some changes from ef-tests-electra * cargo fmt * fix failing test * Revert dockerfile changes * make committee_index return option * function args shouldnt be a ref to attestation ref * fmt * fix dup imports --------- Co-authored-by: realbigsean --- .../src/attestation_verification.rs | 189 ++++++++---- .../src/attestation_verification/batch.rs | 1 - beacon_node/beacon_chain/src/beacon_chain.rs | 29 +- .../beacon_chain/src/early_attester_cache.rs | 1 - beacon_node/beacon_chain/src/errors.rs | 3 + .../src/naive_aggregation_pool.rs | 6 +- beacon_node/beacon_chain/src/test_utils.rs | 280 +++++++++++++----- .../tests/attestation_verification.rs | 48 ++- .../beacon_chain/tests/block_verification.rs | 83 ++++-- .../tests/payload_invalidation.rs | 5 +- beacon_node/beacon_chain/tests/store_tests.rs | 33 ++- .../tests/sync_committee_verification.rs | 3 +- beacon_node/http_api/src/lib.rs | 4 +- .../http_api/src/publish_attestations.rs | 2 +- .../lighthouse_network/src/types/pubsub.rs | 8 +- .../gossip_methods.rs | 34 +++ beacon_node/operation_pool/src/attestation.rs | 20 +- .../operation_pool/src/attestation_storage.rs | 118 ++++---- beacon_node/operation_pool/src/lib.rs | 12 +- consensus/fork_choice/tests/tests.rs | 7 +- .../src/common/get_attesting_indices.rs | 97 +++++- .../per_block_processing/signature_sets.rs | 2 +- consensus/types/src/attestation.rs | 44 ++- consensus/types/src/beacon_state.rs | 6 + consensus/types/src/indexed_attestation.rs | 3 + .../types/src/signed_aggregate_and_proof.rs | 1 - consensus/types/src/subnet_id.rs | 6 +- validator_client/src/attestation_service.rs | 10 +- 28 files changed, 764 insertions(+), 291 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 9c44c5529b3..932374d4867 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -45,7 +45,10 @@ use proto_array::Block as ProtoBlock; use slog::debug; use slot_clock::SlotClock; use state_processing::{ - common::{attesting_indices_base, attesting_indices_electra}, + common::{ + attesting_indices_base, + attesting_indices_electra::{self, get_committee_indices}, + }, per_block_processing::errors::{AttestationValidationError, BlockOperationError}, signature_sets::{ indexed_attestation_signature_set_from_pubkeys, @@ -55,10 +58,11 @@ use state_processing::{ use std::borrow::Cow; use strum::AsRefStr; use tree_hash::TreeHash; +use tree_hash_derive::TreeHash; use types::{ - Attestation, AttestationRef, BeaconCommittee, BeaconStateError::NoCommitteeFound, ChainSpec, - CommitteeIndex, Epoch, EthSpec, ForkName, Hash256, IndexedAttestation, SelectionProof, - SignedAggregateAndProof, Slot, SubnetId, + Attestation, AttestationData, AttestationRef, BeaconCommittee, BeaconStateError, + BeaconStateError::NoCommitteeFound, ChainSpec, CommitteeIndex, Epoch, EthSpec, ForkName, + Hash256, IndexedAttestation, SelectionProof, SignedAggregateAndProof, Slot, SubnetId, }; pub use batch::{batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations}; @@ -139,6 +143,12 @@ pub enum Error { /// /// The peer has sent an invalid message. ValidatorIndexTooHigh(usize), + /// The validator index is not set to zero after Electra. + /// + /// ## Peer scoring + /// + /// The peer has sent an invalid message. + CommitteeIndexNonZero(usize), /// The `attestation.data.beacon_block_root` block is unknown. /// /// ## Peer scoring @@ -187,6 +197,12 @@ pub enum Error { /// /// The peer has sent an invalid message. NotExactlyOneAggregationBitSet(usize), + /// The attestation doesn't have only one aggregation bit set. + /// + /// ## Peer scoring + /// + /// The peer has sent an invalid message. + NotExactlyOneCommitteeBitSet(usize), /// We have already observed an attestation for the `validator_index` and refuse to process /// another. /// @@ -248,9 +264,30 @@ pub enum Error { BeaconChainError(BeaconChainError), } +// TODO(electra) the error conversion changes here are to get a test case to pass +// this could easily be cleaned up impl From for Error { fn from(e: BeaconChainError) -> Self { - Error::BeaconChainError(e) + match &e { + BeaconChainError::BeaconStateError(beacon_state_error) => { + if let BeaconStateError::AggregatorNotInCommittee { aggregator_index } = + beacon_state_error + { + Self::AggregatorNotInCommittee { + aggregator_index: *aggregator_index, + } + } else if let BeaconStateError::InvalidSelectionProof { aggregator_index } = + beacon_state_error + { + Self::InvalidSelectionProof { + aggregator_index: *aggregator_index, + } + } else { + Error::BeaconChainError(e) + } + } + _ => Error::BeaconChainError(e), + } } } @@ -265,10 +302,17 @@ enum CheckAttestationSignature { /// `IndexedAttestation` can be derived. /// /// These attestations have *not* undergone signature verification. +/// The `observed_attestation_key_root` is the hashed value of an `ObservedAttestationKey`. struct IndexedAggregatedAttestation<'a, T: BeaconChainTypes> { signed_aggregate: &'a SignedAggregateAndProof, indexed_attestation: IndexedAttestation, - attestation_data_root: Hash256, + observed_attestation_key_root: Hash256, +} + +#[derive(TreeHash)] +pub struct ObservedAttestationKey { + pub committee_index: u64, + pub attestation_data: AttestationData, } /// Wraps a `Attestation` that has been verified up until the point that an `IndexedAttestation` can @@ -466,18 +510,27 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { }); } - // Ensure the valid aggregated attestation has not already been seen locally. - let attestation_data = attestation.data(); - let attestation_data_root = attestation_data.tree_hash_root(); + let observed_attestation_key_root = ObservedAttestationKey { + committee_index: attestation + .committee_index() + .ok_or(Error::NotExactlyOneCommitteeBitSet(0))?, + attestation_data: attestation.data().clone(), + } + .tree_hash_root(); + + // [New in Electra:EIP7549] + verify_committee_index(attestation, &chain.spec)?; if chain .observed_attestations .write() - .is_known_subset(attestation, attestation_data_root) + .is_known_subset(attestation, observed_attestation_key_root) .map_err(|e| Error::BeaconChainError(e.into()))? { metrics::inc_counter(&metrics::AGGREGATED_ATTESTATION_SUBSETS); - return Err(Error::AttestationSupersetKnown(attestation_data_root)); + return Err(Error::AttestationSupersetKnown( + observed_attestation_key_root, + )); } let aggregator_index = signed_aggregate.message().aggregator_index(); @@ -523,7 +576,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { if attestation.is_aggregation_bits_zero() { Err(Error::EmptyAggregationBitfield) } else { - Ok(attestation_data_root) + Ok(observed_attestation_key_root) } } @@ -533,10 +586,8 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { chain: &BeaconChain, ) -> Result> { use AttestationSlashInfo::*; - - let attestation = signed_aggregate.message().aggregate(); - let aggregator_index = signed_aggregate.message().aggregator_index(); - let attestation_data_root = match Self::verify_early_checks(signed_aggregate, chain) { + let observed_attestation_key_root = match Self::verify_early_checks(signed_aggregate, chain) + { Ok(root) => root, Err(e) => { return Err(SignatureNotChecked( @@ -545,11 +596,12 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { )) } }; - let get_indexed_attestation_with_committee = |(committees, _): (Vec, CommitteesPerSlot)| { - match attestation { - AttestationRef::Base(att) => { + match signed_aggregate { + SignedAggregateAndProof::Base(signed_aggregate) => { + let att = &signed_aggregate.message.aggregate; + let aggregator_index = signed_aggregate.message.aggregator_index; let committee = committees .iter() .filter(|&committee| committee.index == att.data.index) @@ -559,13 +611,13 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { index: att.data.index, })?; + // TODO(electra): + // Note: this clones the signature which is known to be a relatively slow operation. + // + // Future optimizations should remove this clone. if let Some(committee) = committee { - // TODO(electra): - // Note: this clones the signature which is known to be a relatively slow operation. - // - // Future optimizations should remove this clone. let selection_proof = SelectionProof::from( - signed_aggregate.message().selection_proof().clone(), + signed_aggregate.message.selection_proof.clone(), ); if !selection_proof @@ -579,6 +631,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { if !committee.committee.contains(&(aggregator_index as usize)) { return Err(Error::AggregatorNotInCommittee { aggregator_index }); } + attesting_indices_base::get_indexed_attestation( committee.committee, att, @@ -591,13 +644,18 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { }) } } - AttestationRef::Electra(att) => { - attesting_indices_electra::get_indexed_attestation(&committees, att) - .map_err(|e| BeaconChainError::from(e).into()) + SignedAggregateAndProof::Electra(signed_aggregate) => { + attesting_indices_electra::get_indexed_attestation_from_signed_aggregate( + &committees, + signed_aggregate, + &chain.spec, + ) + .map_err(|e| BeaconChainError::from(e).into()) } } }; + let attestation = signed_aggregate.message().aggregate(); let indexed_attestation = match map_attestation_committees( chain, attestation, @@ -611,11 +669,10 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { )) } }; - Ok(IndexedAggregatedAttestation { signed_aggregate, indexed_attestation, - attestation_data_root, + observed_attestation_key_root, }) } } @@ -624,7 +681,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { /// Run the checks that happen after the indexed attestation and signature have been checked. fn verify_late_checks( signed_aggregate: &SignedAggregateAndProof, - attestation_data_root: Hash256, + observed_attestation_key_root: Hash256, chain: &BeaconChain, ) -> Result<(), Error> { let attestation = signed_aggregate.message().aggregate(); @@ -637,11 +694,13 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { if let ObserveOutcome::Subset = chain .observed_attestations .write() - .observe_item(attestation, Some(attestation_data_root)) + .observe_item(attestation, Some(observed_attestation_key_root)) .map_err(|e| Error::BeaconChainError(e.into()))? { metrics::inc_counter(&metrics::AGGREGATED_ATTESTATION_SUBSETS); - return Err(Error::AttestationSupersetKnown(attestation_data_root)); + return Err(Error::AttestationSupersetKnown( + observed_attestation_key_root, + )); } // Observe the aggregator so we don't process another aggregate from them. @@ -701,7 +760,7 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { let IndexedAggregatedAttestation { signed_aggregate, indexed_attestation, - attestation_data_root, + observed_attestation_key_root, } = signed_aggregate; match check_signature { @@ -725,7 +784,9 @@ impl<'a, T: BeaconChainTypes> VerifiedAggregatedAttestation<'a, T> { CheckAttestationSignature::No => (), }; - if let Err(e) = Self::verify_late_checks(signed_aggregate, attestation_data_root, chain) { + if let Err(e) = + Self::verify_late_checks(signed_aggregate, observed_attestation_key_root, chain) + { return Err(SignatureValid(indexed_attestation, e)); } @@ -775,6 +836,9 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { return Err(Error::NotExactlyOneAggregationBitSet(num_aggregation_bits)); } + // [New in Electra:EIP7549] + verify_committee_index(attestation, &chain.spec)?; + // Attestations must be for a known block. If the block is unknown, we simply drop the // attestation and do not delay consideration for later. // @@ -797,7 +861,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { chain: &BeaconChain, ) -> Result<(u64, SubnetId), Error> { let expected_subnet_id = SubnetId::compute_subnet_for_attestation::( - &attestation, + attestation, committees_per_slot, &chain.spec, ) @@ -1101,14 +1165,13 @@ pub fn verify_propagation_slot_range( let current_fork = spec.fork_name_at_slot::(slot_clock.now().ok_or(BeaconChainError::UnableToReadSlot)?); - let earliest_permissible_slot = match current_fork { - ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { - one_epoch_prior - } - // EIP-7045 - ForkName::Deneb | ForkName::Electra => one_epoch_prior + let earliest_permissible_slot = if current_fork < ForkName::Deneb { + one_epoch_prior + // EIP-7045 + } else { + one_epoch_prior .epoch(E::slots_per_epoch()) - .start_slot(E::slots_per_epoch()), + .start_slot(E::slots_per_epoch()) }; if attestation_slot < earliest_permissible_slot { @@ -1147,7 +1210,6 @@ pub fn verify_attestation_signature( &chain.spec, ) .map_err(BeaconChainError::SignatureSetError)?; - metrics::stop_timer(signature_setup_timer); let _signature_verification_timer = @@ -1273,6 +1335,35 @@ pub fn verify_signed_aggregate_signatures( Ok(verify_signature_sets(signature_sets.iter())) } +/// Verify that the `attestation` committee index is properly set for the attestation's fork. +/// This function will only apply verification post-Electra. +pub fn verify_committee_index( + attestation: AttestationRef, + spec: &ChainSpec, +) -> Result<(), Error> { + if spec.fork_name_at_slot::(attestation.data().slot) >= ForkName::Electra { + // Check to ensure that the attestation is for a single committee. + let num_committee_bits = get_committee_indices::( + attestation + .committee_bits() + .map_err(|e| Error::BeaconChainError(e.into()))?, + ); + if num_committee_bits.len() != 1 { + return Err(Error::NotExactlyOneCommitteeBitSet( + num_committee_bits.len(), + )); + } + + // Ensure the attestation index is set to zero post Electra. + if attestation.data().index != 0 { + return Err(Error::CommitteeIndexNonZero( + attestation.data().index as usize, + )); + } + } + Ok(()) +} + /// Assists in readability. type CommitteesPerSlot = u64; @@ -1309,7 +1400,7 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( attesting_indices_electra::get_indexed_attestation(&committees, att) .map(|attestation| (attestation, committees_per_slot)) .map_err(|e| { - let index = att.committee_index(); + let index = att.committee_index().unwrap_or(0); if e == BlockOperationError::BeaconStateError(NoCommitteeFound(index)) { Error::NoCommitteeForSlotAndIndex { slot: att.data.slot, @@ -1324,16 +1415,14 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( }) } -// TODO(electra) update comments below to reflect logic changes -// i.e. this now runs the map_fn on a list of committees for the slot of the provided attestation /// Runs the `map_fn` with the committee and committee count per slot for the given `attestation`. /// -/// This function exists in this odd "map" pattern because efficiently obtaining the committee for -/// an attestation can be complex. It might involve reading straight from the +/// This function exists in this odd "map" pattern because efficiently obtaining the committees for +/// an attestations slot can be complex. It might involve reading straight from the /// `beacon_chain.shuffling_cache` or it might involve reading it from a state from the DB. Due to /// the complexities of `RwLock`s on the shuffling cache, a simple `Cow` isn't suitable here. /// -/// If the committee for `attestation` isn't found in the `shuffling_cache`, we will read a state +/// If the committees for an `attestation`'s slot isn't found in the `shuffling_cache`, we will read a state /// from disk and then update the `shuffling_cache`. fn map_attestation_committees( chain: &BeaconChain, @@ -1373,7 +1462,7 @@ where .unwrap_or_else(|_| { Err(Error::NoCommitteeForSlotAndIndex { slot: attestation.data().slot, - index: attestation.committee_index(), + index: attestation.committee_index().unwrap_or(0), }) })) }) diff --git a/beacon_node/beacon_chain/src/attestation_verification/batch.rs b/beacon_node/beacon_chain/src/attestation_verification/batch.rs index 1ec752ff5c5..07fad1bd4a8 100644 --- a/beacon_node/beacon_chain/src/attestation_verification/batch.rs +++ b/beacon_node/beacon_chain/src/attestation_verification/batch.rs @@ -66,7 +66,6 @@ where .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)?; let mut signature_sets = Vec::with_capacity(num_indexed * 3); - // Iterate, flattening to get only the `Ok` values. for indexed in indexing_results.iter().flatten() { let signed_aggregate = &indexed.signed_aggregate; diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d0c89581323..1428848fbc8 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -85,7 +85,9 @@ use futures::channel::mpsc::Sender; use itertools::process_results; use itertools::Itertools; use kzg::Kzg; -use operation_pool::{AttestationRef, OperationPool, PersistedOperationPool, ReceivedPreCapella}; +use operation_pool::{ + CompactAttestationRef, OperationPool, PersistedOperationPool, ReceivedPreCapella, +}; use parking_lot::{Mutex, RwLock}; use proto_array::{DoNotReOrg, ProposerHeadError}; use safe_arith::SafeArith; @@ -1609,6 +1611,21 @@ impl BeaconChain { Ok((duties, dependent_root, execution_status)) } + pub fn get_aggregated_attestation( + &self, + attestation: AttestationRef, + ) -> Result>, Error> { + match attestation { + AttestationRef::Base(att) => self.get_aggregated_attestation_base(&att.data), + AttestationRef::Electra(att) => self.get_aggregated_attestation_electra( + att.data.slot, + &att.data.tree_hash_root(), + att.committee_index() + .ok_or(Error::AttestationCommitteeIndexNotSet)?, + ), + } + } + /// Returns an aggregated `Attestation`, if any, that has a matching `attestation.data`. /// /// The attestation will be obtained from `self.naive_aggregation_pool`. @@ -2185,7 +2202,7 @@ impl BeaconChain { self.log, "Stored unaggregated attestation"; "outcome" => ?outcome, - "index" => attestation.data().index, + "index" => attestation.committee_index(), "slot" => attestation.data().slot.as_u64(), ), Err(NaiveAggregationError::SlotTooLow { @@ -2204,7 +2221,7 @@ impl BeaconChain { self.log, "Failed to store unaggregated attestation"; "error" => ?e, - "index" => attestation.data().index, + "index" => attestation.committee_index(), "slot" => attestation.data().slot.as_u64(), ); return Err(Error::from(e).into()); @@ -2329,7 +2346,7 @@ impl BeaconChain { pub fn filter_op_pool_attestation( &self, filter_cache: &mut HashMap<(Hash256, Epoch), bool>, - att: &AttestationRef, + att: &CompactAttestationRef, state: &BeaconState, ) -> bool { *filter_cache @@ -4948,11 +4965,11 @@ impl BeaconChain { initialize_epoch_cache(&mut state, &self.spec)?; let mut prev_filter_cache = HashMap::new(); - let prev_attestation_filter = |att: &AttestationRef| { + let prev_attestation_filter = |att: &CompactAttestationRef| { self.filter_op_pool_attestation(&mut prev_filter_cache, att, &state) }; let mut curr_filter_cache = HashMap::new(); - let curr_attestation_filter = |att: &AttestationRef| { + let curr_attestation_filter = |att: &CompactAttestationRef| { self.filter_op_pool_attestation(&mut curr_filter_cache, att, &state) }; diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index 8ed4e5db40b..936f4de3ee1 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -123,7 +123,6 @@ impl EarlyAttesterCache { item.committee_lengths .get_committee_length::(request_slot, request_index, spec)?; - // TODO(electra) make fork-agnostic let attestation = if spec.fork_name_at_slot::(request_slot) >= ForkName::Electra { let mut committee_bits = BitVector::default(); if committee_len > 0 { diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 340f1f9f797..3d61d4f32d7 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -226,6 +226,8 @@ pub enum BeaconChainError { LightClientError(LightClientError), UnsupportedFork, MilhouseError(MilhouseError), + AttestationError(AttestationError), + AttestationCommitteeIndexNotSet, } easy_from_to!(SlotProcessingError, BeaconChainError); @@ -256,6 +258,7 @@ easy_from_to!(AvailabilityCheckError, BeaconChainError); easy_from_to!(EpochCacheError, BeaconChainError); easy_from_to!(LightClientError, BeaconChainError); easy_from_to!(MilhouseError, BeaconChainError); +easy_from_to!(AttestationError, BeaconChainError); #[derive(Debug)] pub enum BlockProductionError { diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index a1c736cd0ef..b16ced789ff 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -261,9 +261,9 @@ impl AggregateMap for AggregatedAttestationMap { }; let attestation_key = AttestationKey::from_attestation_ref(a)?; - let attestation_data_root = attestation_key.tree_hash_root(); + let attestation_key_root = attestation_key.tree_hash_root(); - if let Some(existing_attestation) = self.map.get_mut(&attestation_data_root) { + if let Some(existing_attestation) = self.map.get_mut(&attestation_key_root) { if existing_attestation .get_aggregation_bit(aggregation_bit) .map_err(|_| Error::InconsistentBitfieldLengths)? @@ -285,7 +285,7 @@ impl AggregateMap for AggregatedAttestationMap { } self.map - .insert(attestation_data_root, a.clone_as_attestation()); + .insert(attestation_key_root, a.clone_as_attestation()); Ok(InsertOutcome::NewItemInserted { committee_index: aggregation_bit, }) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 09e90093816..d5716130cc9 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1169,7 +1169,7 @@ where }; let subnet_id = SubnetId::compute_subnet_for_attestation::( - &attestation.to_ref(), + attestation.to_ref(), committee_count, &self.chain.spec, ) @@ -1343,7 +1343,10 @@ where // If there are any attestations in this committee, create an aggregate. if let Some((attestation, _)) = committee_attestations.first() { let bc = state - .get_beacon_committee(attestation.data().slot, attestation.data().index) + .get_beacon_committee( + attestation.data().slot, + attestation.committee_index().unwrap(), + ) .unwrap(); // Find an aggregator if one exists. Return `None` if there are no @@ -1370,21 +1373,42 @@ where }) .copied()?; + let fork_name = self.spec.fork_name_at_slot::(slot); + + let aggregate = if fork_name >= ForkName::Electra { + self.chain + .get_aggregated_attestation_electra( + slot, + &attestation.data().tree_hash_root(), + bc.index, + ) + .unwrap() + .unwrap_or_else(|| { + committee_attestations.iter().skip(1).fold( + attestation.clone(), + |mut agg, (att, _)| { + agg.aggregate(att.to_ref()); + agg + }, + ) + }) + } else { + self.chain + .get_aggregated_attestation_base(attestation.data()) + .unwrap() + .unwrap_or_else(|| { + committee_attestations.iter().skip(1).fold( + attestation.clone(), + |mut agg, (att, _)| { + agg.aggregate(att.to_ref()); + agg + }, + ) + }) + }; + // If the chain is able to produce an aggregate, use that. Otherwise, build an // aggregate locally. - let aggregate = self - .chain - .get_aggregated_attestation_base(attestation.data()) - .unwrap() - .unwrap_or_else(|| { - committee_attestations.iter().skip(1).fold( - attestation.clone(), - |mut agg, (att, _)| { - agg.aggregate(att.to_ref()); - agg - }, - ) - }); let signed_aggregate = SignedAggregateAndProof::from_aggregate( aggregator_index as u64, @@ -1514,54 +1538,101 @@ where ) -> AttesterSlashing { let fork = self.chain.canonical_head.cached_head().head_fork(); - // TODO(electra): consider making this test fork-agnostic - let mut attestation_1 = IndexedAttestationBase { - attesting_indices: VariableList::new(validator_indices).unwrap(), - data: AttestationData { - slot: Slot::new(0), - index: 0, - beacon_block_root: Hash256::zero(), - target: Checkpoint { - root: Hash256::zero(), - epoch: target1.unwrap_or(fork.epoch), + let fork_name = self.spec.fork_name_at_slot::(Slot::new(0)); + + let mut attestation_1 = if fork_name >= ForkName::Electra { + IndexedAttestation::Electra(IndexedAttestationElectra { + attesting_indices: VariableList::new(validator_indices).unwrap(), + data: AttestationData { + slot: Slot::new(0), + index: 0, + beacon_block_root: Hash256::zero(), + target: Checkpoint { + root: Hash256::zero(), + epoch: target1.unwrap_or(fork.epoch), + }, + source: Checkpoint { + root: Hash256::zero(), + epoch: source1.unwrap_or(Epoch::new(0)), + }, }, - source: Checkpoint { - root: Hash256::zero(), - epoch: source1.unwrap_or(Epoch::new(0)), + signature: AggregateSignature::infinity(), + }) + } else { + IndexedAttestation::Base(IndexedAttestationBase { + attesting_indices: VariableList::new(validator_indices).unwrap(), + data: AttestationData { + slot: Slot::new(0), + index: 0, + beacon_block_root: Hash256::zero(), + target: Checkpoint { + root: Hash256::zero(), + epoch: target1.unwrap_or(fork.epoch), + }, + source: Checkpoint { + root: Hash256::zero(), + epoch: source1.unwrap_or(Epoch::new(0)), + }, }, - }, - signature: AggregateSignature::infinity(), + signature: AggregateSignature::infinity(), + }) }; let mut attestation_2 = attestation_1.clone(); - attestation_2.data.index += 1; - attestation_2.data.source.epoch = source2.unwrap_or(Epoch::new(0)); - attestation_2.data.target.epoch = target2.unwrap_or(fork.epoch); + attestation_2.data_mut().index += 1; + attestation_2.data_mut().source.epoch = source2.unwrap_or(Epoch::new(0)); + attestation_2.data_mut().target.epoch = target2.unwrap_or(fork.epoch); for attestation in &mut [&mut attestation_1, &mut attestation_2] { - // TODO(electra) we could explore iter mut here - for i in attestation.attesting_indices.iter() { - let sk = &self.validator_keypairs[*i as usize].sk; + match attestation { + IndexedAttestation::Base(attestation) => { + for i in attestation.attesting_indices.iter() { + let sk = &self.validator_keypairs[*i as usize].sk; - let genesis_validators_root = self.chain.genesis_validators_root; + let genesis_validators_root = self.chain.genesis_validators_root; - let domain = self.chain.spec.get_domain( - attestation.data.target.epoch, - Domain::BeaconAttester, - &fork, - genesis_validators_root, - ); - let message = attestation.data.signing_root(domain); + let domain = self.chain.spec.get_domain( + attestation.data.target.epoch, + Domain::BeaconAttester, + &fork, + genesis_validators_root, + ); + let message = attestation.data.signing_root(domain); + + attestation.signature.add_assign(&sk.sign(message)); + } + } + IndexedAttestation::Electra(attestation) => { + for i in attestation.attesting_indices.iter() { + let sk = &self.validator_keypairs[*i as usize].sk; - attestation.signature.add_assign(&sk.sign(message)); + let genesis_validators_root = self.chain.genesis_validators_root; + + let domain = self.chain.spec.get_domain( + attestation.data.target.epoch, + Domain::BeaconAttester, + &fork, + genesis_validators_root, + ); + let message = attestation.data.signing_root(domain); + + attestation.signature.add_assign(&sk.sign(message)); + } + } } } - // TODO(electra): fix this test - AttesterSlashing::Base(AttesterSlashingBase { - attestation_1, - attestation_2, - }) + if fork_name >= ForkName::Electra { + AttesterSlashing::Electra(AttesterSlashingElectra { + attestation_1: attestation_1.as_electra().unwrap().clone(), + attestation_2: attestation_2.as_electra().unwrap().clone(), + }) + } else { + AttesterSlashing::Base(AttesterSlashingBase { + attestation_1: attestation_1.as_base().unwrap().clone(), + attestation_2: attestation_2.as_base().unwrap().clone(), + }) + } } pub fn make_attester_slashing_different_indices( @@ -1569,6 +1640,8 @@ where validator_indices_1: Vec, validator_indices_2: Vec, ) -> AttesterSlashing { + let fork_name = self.spec.fork_name_at_slot::(Slot::new(0)); + let data = AttestationData { slot: Slot::new(0), index: 0, @@ -1583,45 +1656,95 @@ where }, }; - // TODO(electra): make this test fork-agnostic - let mut attestation_1 = IndexedAttestationBase { - attesting_indices: VariableList::new(validator_indices_1).unwrap(), - data: data.clone(), - signature: AggregateSignature::infinity(), - }; + let (mut attestation_1, mut attestation_2) = if fork_name >= ForkName::Electra { + let attestation_1 = IndexedAttestationElectra { + attesting_indices: VariableList::new(validator_indices_1).unwrap(), + data: data.clone(), + signature: AggregateSignature::infinity(), + }; + + let attestation_2 = IndexedAttestationElectra { + attesting_indices: VariableList::new(validator_indices_2).unwrap(), + data, + signature: AggregateSignature::infinity(), + }; - let mut attestation_2 = IndexedAttestationBase { - attesting_indices: VariableList::new(validator_indices_2).unwrap(), - data, - signature: AggregateSignature::infinity(), + ( + IndexedAttestation::Electra(attestation_1), + IndexedAttestation::Electra(attestation_2), + ) + } else { + let attestation_1 = IndexedAttestationBase { + attesting_indices: VariableList::new(validator_indices_1).unwrap(), + data: data.clone(), + signature: AggregateSignature::infinity(), + }; + + let attestation_2 = IndexedAttestationBase { + attesting_indices: VariableList::new(validator_indices_2).unwrap(), + data, + signature: AggregateSignature::infinity(), + }; + + ( + IndexedAttestation::Base(attestation_1), + IndexedAttestation::Base(attestation_2), + ) }; - attestation_2.data.index += 1; + attestation_2.data_mut().index += 1; let fork = self.chain.canonical_head.cached_head().head_fork(); for attestation in &mut [&mut attestation_1, &mut attestation_2] { - for i in attestation.attesting_indices.iter() { - let sk = &self.validator_keypairs[*i as usize].sk; + match attestation { + IndexedAttestation::Base(attestation) => { + for i in attestation.attesting_indices.iter() { + let sk = &self.validator_keypairs[*i as usize].sk; + + let genesis_validators_root = self.chain.genesis_validators_root; + + let domain = self.chain.spec.get_domain( + attestation.data.target.epoch, + Domain::BeaconAttester, + &fork, + genesis_validators_root, + ); + let message = attestation.data.signing_root(domain); - let genesis_validators_root = self.chain.genesis_validators_root; + attestation.signature.add_assign(&sk.sign(message)); + } + } + IndexedAttestation::Electra(attestation) => { + for i in attestation.attesting_indices.iter() { + let sk = &self.validator_keypairs[*i as usize].sk; - let domain = self.chain.spec.get_domain( - attestation.data.target.epoch, - Domain::BeaconAttester, - &fork, - genesis_validators_root, - ); - let message = attestation.data.signing_root(domain); + let genesis_validators_root = self.chain.genesis_validators_root; - attestation.signature.add_assign(&sk.sign(message)); + let domain = self.chain.spec.get_domain( + attestation.data.target.epoch, + Domain::BeaconAttester, + &fork, + genesis_validators_root, + ); + let message = attestation.data.signing_root(domain); + + attestation.signature.add_assign(&sk.sign(message)); + } + } } } - // TODO(electra): fix this test - AttesterSlashing::Base(AttesterSlashingBase { - attestation_1, - attestation_2, - }) + if fork_name >= ForkName::Electra { + AttesterSlashing::Electra(AttesterSlashingElectra { + attestation_1: attestation_1.as_electra().unwrap().clone(), + attestation_2: attestation_2.as_electra().unwrap().clone(), + }) + } else { + AttesterSlashing::Base(AttesterSlashingBase { + attestation_1: attestation_1.as_base().unwrap().clone(), + attestation_2: attestation_2.as_base().unwrap().clone(), + }) + } } pub fn make_proposer_slashing(&self, validator_index: u64) -> ProposerSlashing { @@ -2413,6 +2536,7 @@ where AttestationStrategy::AllValidators => self.get_all_validators(), AttestationStrategy::SomeValidators(vals) => vals, }; + let state_root = state.update_tree_hash_cache().unwrap(); let (_, _, last_produced_block_hash, _) = self .add_attested_blocks_at_slots_with_sync( @@ -2539,6 +2663,7 @@ pub fn generate_rand_block_and_blobs( rng: &mut impl Rng, ) -> (SignedBeaconBlock>, Vec>) { let inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(rng)); + let mut block = SignedBeaconBlock::from_block(inner, types::Signature::random_for_test(rng)); let mut blob_sidecars = vec![]; @@ -2575,7 +2700,6 @@ pub fn generate_rand_block_and_blobs( }; let (bundle, transactions) = execution_layer::test_utils::generate_blobs::(num_blobs).unwrap(); - payload.execution_payload.transactions = <_>::default(); for tx in Vec::from(transactions) { payload.execution_payload.transactions.push(tx).unwrap(); diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index e91e8c77a3c..63740c4736c 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -2,6 +2,7 @@ use beacon_chain::attestation_verification::{ batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations, Error, + ObservedAttestationKey, }; use beacon_chain::test_utils::{MakeAttestationOptions, HARNESS_GENESIS_TIME}; use beacon_chain::{ @@ -128,7 +129,12 @@ fn get_valid_unaggregated_attestation( let validator_committee_index = 0; let validator_index = *head .beacon_state - .get_beacon_committee(current_slot, valid_attestation.data().index) + .get_beacon_committee( + current_slot, + valid_attestation + .committee_index() + .expect("should get committee index"), + ) .expect("should get committees") .committee .get(validator_committee_index) @@ -146,8 +152,8 @@ fn get_valid_unaggregated_attestation( ) .expect("should sign attestation"); - let subnet_id = SubnetId::compute_subnet_for_attestation::( - &valid_attestation.to_ref(), + let subnet_id = SubnetId::compute_subnet_for_attestation::( + valid_attestation.to_ref(), head.beacon_state .get_committee_count_at_slot(current_slot) .expect("should get committee count"), @@ -173,7 +179,12 @@ fn get_valid_aggregated_attestation( let current_slot = chain.slot().expect("should get slot"); let committee = state - .get_beacon_committee(current_slot, aggregate.data().index) + .get_beacon_committee( + current_slot, + aggregate + .committee_index() + .expect("should get committee index"), + ) .expect("should get committees"); let committee_len = committee.committee.len(); @@ -216,7 +227,7 @@ fn get_valid_aggregated_attestation( /// attestation. fn get_non_aggregator( chain: &BeaconChain, - aggregate: &AttestationRef, + aggregate: AttestationRef, ) -> (usize, SecretKey) { let head = chain.head_snapshot(); let state = &head.beacon_state; @@ -224,7 +235,12 @@ fn get_non_aggregator( // TODO(electra) make fork-agnostic let committee = state - .get_beacon_committee(current_slot, aggregate.data().index) + .get_beacon_committee( + current_slot, + aggregate + .committee_index() + .expect("should get committee index"), + ) .expect("should get committees"); let committee_len = committee.committee.len(); @@ -371,7 +387,7 @@ impl GossipTester { pub fn non_aggregator(&self) -> (usize, SecretKey) { get_non_aggregator( &self.harness.chain, - &self.valid_aggregate.message().aggregate(), + self.valid_aggregate.message().aggregate(), ) } @@ -429,6 +445,7 @@ impl GossipTester { vec![&self.invalid_aggregate, &aggregate].into_iter(), ) .unwrap(); + assert_eq!(results.len(), 2); let batch_err = results.pop().unwrap().err().expect(&format!( "{} should error during batch_verify_aggregated_attestations_for_gossip", @@ -662,7 +679,7 @@ async fn aggregated_gossip_verification() { .chain .head_snapshot() .beacon_state - .get_beacon_committee(tester.slot(), a.message().aggregate().data().index) + .get_beacon_committee(tester.slot(), a.message().aggregate().committee_index().expect("should get committee index")) .expect("should get committees") .committee .len(); @@ -778,12 +795,7 @@ async fn aggregated_gossip_verification() { // However, the following error is triggered first: AttnError::AggregatorNotInCommittee { aggregator_index - } | - // unless were working with electra attestations - // in which case this error is triggered instead: - AttnError::AggregatorPubkeyUnknown( - aggregator_index - ) + } if aggregator_index == VALIDATOR_COUNT as u64 )) }, @@ -792,7 +804,7 @@ async fn aggregated_gossip_verification() { * The following test ensures: * * aggregate_and_proof.selection_proof selects the validator as an aggregator for the slot -- - * i.e. is_aggregator(state, aggregate.data.slot, aggregate.data.index, + * i.e. is_aggregator(state, aggregate.data.slot, aggregate.committee_index(), * aggregate_and_proof.selection_proof) returns True. */ .inspect_aggregate_err( @@ -812,6 +824,7 @@ async fn aggregated_gossip_verification() { }, |tester, err| { let (val_index, _) = tester.non_aggregator(); + assert!(matches!( err, AttnError::InvalidSelectionProof { @@ -838,7 +851,10 @@ async fn aggregated_gossip_verification() { assert!(matches!( err, AttnError::AttestationSupersetKnown(hash) - if hash == tester.valid_aggregate.message().aggregate().data().tree_hash_root() + if hash == ObservedAttestationKey { + committee_index: tester.valid_aggregate.message().aggregate().expect("should get committee index"), + attestation_data: tester.valid_aggregate.message().aggregate().data().clone(), + }.tree_hash_root() )) }, ) diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 261e15ba922..0fe406a0172 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -666,27 +666,58 @@ async fn invalid_signature_attester_slashing() { for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; let mut snapshots = chain_segment.clone(); - let indexed_attestation = IndexedAttestationBase { - attesting_indices: vec![0].into(), - data: AttestationData { - slot: Slot::new(0), - index: 0, - beacon_block_root: Hash256::zero(), - source: Checkpoint { - epoch: Epoch::new(0), - root: Hash256::zero(), + let fork_name = harness.chain.spec.fork_name_at_slot::(Slot::new(0)); + + let attester_slashing = if fork_name >= ForkName::Electra { + let indexed_attestation = IndexedAttestationElectra { + attesting_indices: vec![0].into(), + data: AttestationData { + slot: Slot::new(0), + index: 0, + beacon_block_root: Hash256::zero(), + source: Checkpoint { + epoch: Epoch::new(0), + root: Hash256::zero(), + }, + target: Checkpoint { + epoch: Epoch::new(0), + root: Hash256::zero(), + }, }, - target: Checkpoint { - epoch: Epoch::new(0), - root: Hash256::zero(), + signature: junk_aggregate_signature(), + }; + let attester_slashing = AttesterSlashingElectra { + attestation_1: indexed_attestation.clone(), + attestation_2: indexed_attestation, + }; + + AttesterSlashing::Electra(attester_slashing) + } else { + let indexed_attestation = IndexedAttestationBase { + attesting_indices: vec![0].into(), + data: AttestationData { + slot: Slot::new(0), + index: 0, + beacon_block_root: Hash256::zero(), + source: Checkpoint { + epoch: Epoch::new(0), + root: Hash256::zero(), + }, + target: Checkpoint { + epoch: Epoch::new(0), + root: Hash256::zero(), + }, }, - }, - signature: junk_aggregate_signature(), - }; - let attester_slashing = AttesterSlashingBase { - attestation_1: indexed_attestation.clone(), - attestation_2: indexed_attestation, + signature: junk_aggregate_signature(), + }; + let attester_slashing = AttesterSlashingBase { + attestation_1: indexed_attestation.clone(), + attestation_2: indexed_attestation, + }; + + AttesterSlashing::Base(attester_slashing) }; + let (mut block, signature) = snapshots[block_index] .beacon_block .as_ref() @@ -695,31 +726,33 @@ async fn invalid_signature_attester_slashing() { match &mut block.body_mut() { BeaconBlockBodyRefMut::Base(ref mut blk) => { blk.attester_slashings - .push(attester_slashing) + .push(attester_slashing.as_base().unwrap().clone()) .expect("should update attester slashing"); } BeaconBlockBodyRefMut::Altair(ref mut blk) => { blk.attester_slashings - .push(attester_slashing) + .push(attester_slashing.as_base().unwrap().clone()) .expect("should update attester slashing"); } BeaconBlockBodyRefMut::Bellatrix(ref mut blk) => { blk.attester_slashings - .push(attester_slashing) + .push(attester_slashing.as_base().unwrap().clone()) .expect("should update attester slashing"); } BeaconBlockBodyRefMut::Capella(ref mut blk) => { blk.attester_slashings - .push(attester_slashing) + .push(attester_slashing.as_base().unwrap().clone()) .expect("should update attester slashing"); } BeaconBlockBodyRefMut::Deneb(ref mut blk) => { blk.attester_slashings - .push(attester_slashing) + .push(attester_slashing.as_base().unwrap().clone()) .expect("should update attester slashing"); } - BeaconBlockBodyRefMut::Electra(_) => { - panic!("electra test not implemented!"); + BeaconBlockBodyRefMut::Electra(ref mut blk) => { + blk.attester_slashings + .push(attester_slashing.as_electra().unwrap().clone()) + .expect("should update attester slashing"); } } snapshots[block_index].beacon_block = diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index bc03767ce5e..4dc7d20e227 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -25,7 +25,6 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use task_executor::ShutdownReason; -use tree_hash::TreeHash; use types::*; const VALIDATOR_COUNT: usize = 32; @@ -1224,13 +1223,13 @@ async fn attesting_to_optimistic_head() { let get_aggregated = || { rig.harness .chain - .get_aggregated_attestation(attestation.data()) + .get_aggregated_attestation(attestation.to_ref()) }; let get_aggregated_by_slot_and_root = || { rig.harness .chain - .get_aggregated_attestation_base(attestation.data()) + .get_aggregated_attestation(attestation.to_ref()) }; /* diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 769d178dbd0..ef873909309 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -1014,6 +1014,7 @@ async fn multiple_attestations_per_block() { .await; let head = harness.chain.head_snapshot(); + let committees_per_slot = head .beacon_state .get_committee_count_at_slot(head.beacon_state.slot()) @@ -1022,15 +1023,29 @@ async fn multiple_attestations_per_block() { for snapshot in harness.chain.chain_dump().unwrap() { let slot = snapshot.beacon_block.slot(); - assert_eq!( - snapshot - .beacon_block - .as_ref() - .message() - .body() - .attestations_len() as u64, - if slot <= 1 { 0 } else { committees_per_slot } - ); + let fork_name = harness.chain.spec.fork_name_at_slot::(slot); + + if fork_name >= ForkName::Electra { + assert_eq!( + snapshot + .beacon_block + .as_ref() + .message() + .body() + .attestations_len() as u64, + if slot <= 1 { 0 } else { 1 } + ); + } else { + assert_eq!( + snapshot + .beacon_block + .as_ref() + .message() + .body() + .attestations_len() as u64, + if slot <= 1 { 0 } else { committees_per_slot } + ); + } } } diff --git a/beacon_node/beacon_chain/tests/sync_committee_verification.rs b/beacon_node/beacon_chain/tests/sync_committee_verification.rs index 0e4745ff6b8..242ed558475 100644 --- a/beacon_node/beacon_chain/tests/sync_committee_verification.rs +++ b/beacon_node/beacon_chain/tests/sync_committee_verification.rs @@ -318,7 +318,6 @@ async fn aggregated_gossip_verification() { * The contribution_and_proof.selection_proof is a valid signature of the `SyncAggregatorSelectionData` * derived from the contribution by the validator with index `contribution_and_proof.aggregator_index`. */ - assert_invalid!( "aggregate with bad selection proof signature", { @@ -354,7 +353,6 @@ async fn aggregated_gossip_verification() { * derived from the participation info in `aggregation_bits` for the subcommittee specified by * the `contribution.subcommittee_index`. */ - assert_invalid!( "aggregate with bad aggregate signature", { @@ -450,6 +448,7 @@ async fn aggregated_gossip_verification() { root: contribution.beacon_block_root, subcommittee_index: contribution.subcommittee_index, }; + assert_invalid!( "aggregate that has already been seen", valid_aggregate.clone(), diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 647fe99645a..6c7126ace53 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3369,7 +3369,7 @@ pub fn serve( "error" => format!("{:?}", e), "request_index" => index, "aggregator_index" => aggregate.message().aggregator_index(), - "attestation_index" => aggregate.message().aggregate().data().index, + "attestation_index" => aggregate.message().aggregate().committee_index(), "attestation_slot" => aggregate.message().aggregate().data().slot, ); failures.push(api_types::Failure::new(index, format!("Verification: {:?}", e))); @@ -3390,7 +3390,7 @@ pub fn serve( "error" => format!("{:?}", e), "request_index" => index, "aggregator_index" => verified_aggregate.aggregate().message().aggregator_index(), - "attestation_index" => verified_aggregate.attestation().data().index, + "attestation_index" => verified_aggregate.attestation().committee_index(), "attestation_slot" => verified_aggregate.attestation().data().slot, ); failures.push(api_types::Failure::new(index, format!("Fork choice: {:?}", e))); diff --git a/beacon_node/http_api/src/publish_attestations.rs b/beacon_node/http_api/src/publish_attestations.rs index 541ba8b7871..00654765325 100644 --- a/beacon_node/http_api/src/publish_attestations.rs +++ b/beacon_node/http_api/src/publish_attestations.rs @@ -141,7 +141,7 @@ pub async fn publish_attestations( // move the `attestations` vec into the blocking task, so this small overhead is unavoidable. let attestation_metadata = attestations .iter() - .map(|att| (att.data().slot, att.data().index)) + .map(|att| (att.data().slot, att.committee_index())) .collect::>(); // Gossip validate and publish attestations that can be immediately processed. diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 346c177e582..7be68f879fe 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -401,17 +401,17 @@ impl std::fmt::Display for PubsubMessage { ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, - "Aggregate and Proof: slot: {}, index: {}, aggregator_index: {}", + "Aggregate and Proof: slot: {}, index: {:?}, aggregator_index: {}", att.message().aggregate().data().slot, - att.message().aggregate().data().index, + att.message().aggregate().committee_index(), att.message().aggregator_index(), ), PubsubMessage::Attestation(data) => write!( f, - "Attestation: subnet_id: {}, attestation_slot: {}, attestation_index: {}", + "Attestation: subnet_id: {}, attestation_slot: {}, attestation_index: {:?}", *data.0, data.1.data().slot, - data.1.data().index, + data.1.committee_index(), ), PubsubMessage::VoluntaryExit(_data) => write!(f, "Voluntary Exit"), PubsubMessage::ProposerSlashing(_data) => write!(f, "Proposer Slashing"), diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index ad1c8867e2f..5498f2e239a 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -2049,6 +2049,27 @@ impl NetworkBeaconProcessor { "attn_val_index_too_high", ); } + AttnError::CommitteeIndexNonZero(index) => { + /* + * The validator index is not set to zero after Electra. + * + * The peer has published an invalid consensus message. + */ + debug!( + self.log, + "Committee index non zero"; + "peer_id" => %peer_id, + "block" => ?beacon_block_root, + "type" => ?attestation_type, + "committee_index" => index, + ); + self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Reject); + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "attn_comm_index_non_zero", + ); + } AttnError::UnknownHeadBlock { beacon_block_root } => { trace!( self.log, @@ -2204,6 +2225,19 @@ impl NetworkBeaconProcessor { "attn_too_many_agg_bits", ); } + AttnError::NotExactlyOneCommitteeBitSet(_) => { + /* + * The attestation doesn't have only one committee bit set. + * + * The peer has published an invalid consensus message. + */ + self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Reject); + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "attn_too_many_comm_bits", + ); + } AttnError::AttestsToFutureBlock { .. } => { /* * The beacon_block_root is from a higher slot than the attestation. diff --git a/beacon_node/operation_pool/src/attestation.rs b/beacon_node/operation_pool/src/attestation.rs index 91fd00a3979..c6ed6eb7f6e 100644 --- a/beacon_node/operation_pool/src/attestation.rs +++ b/beacon_node/operation_pool/src/attestation.rs @@ -1,4 +1,4 @@ -use crate::attestation_storage::{AttestationRef, CompactIndexedAttestation}; +use crate::attestation_storage::{CompactAttestationRef, CompactIndexedAttestation}; use crate::max_cover::MaxCover; use crate::reward_cache::RewardCache; use state_processing::common::{ @@ -14,14 +14,14 @@ use types::{ #[derive(Debug, Clone)] pub struct AttMaxCover<'a, E: EthSpec> { /// Underlying attestation. - pub att: AttestationRef<'a, E>, + pub att: CompactAttestationRef<'a, E>, /// Mapping of validator indices and their rewards. pub fresh_validators_rewards: HashMap, } impl<'a, E: EthSpec> AttMaxCover<'a, E> { pub fn new( - att: AttestationRef<'a, E>, + att: CompactAttestationRef<'a, E>, state: &BeaconState, reward_cache: &'a RewardCache, total_active_balance: u64, @@ -36,7 +36,7 @@ impl<'a, E: EthSpec> AttMaxCover<'a, E> { /// Initialise an attestation cover object for base/phase0 hard fork. pub fn new_for_base( - att: AttestationRef<'a, E>, + att: CompactAttestationRef<'a, E>, state: &BeaconState, base_state: &BeaconStateBase, total_active_balance: u64, @@ -69,7 +69,7 @@ impl<'a, E: EthSpec> AttMaxCover<'a, E> { /// Initialise an attestation cover object for Altair or later. pub fn new_for_altair_deneb( - att: AttestationRef<'a, E>, + att: CompactAttestationRef<'a, E>, state: &BeaconState, reward_cache: &'a RewardCache, spec: &ChainSpec, @@ -119,14 +119,14 @@ impl<'a, E: EthSpec> AttMaxCover<'a, E> { impl<'a, E: EthSpec> MaxCover for AttMaxCover<'a, E> { type Object = Attestation; - type Intermediate = AttestationRef<'a, E>; + type Intermediate = CompactAttestationRef<'a, E>; type Set = HashMap; - fn intermediate(&self) -> &AttestationRef<'a, E> { + fn intermediate(&self) -> &CompactAttestationRef<'a, E> { &self.att } - fn convert_to_object(att_ref: &AttestationRef<'a, E>) -> Attestation { + fn convert_to_object(att_ref: &CompactAttestationRef<'a, E>) -> Attestation { att_ref.clone_as_attestation() } @@ -153,7 +153,7 @@ impl<'a, E: EthSpec> MaxCover for AttMaxCover<'a, E> { /// executing the `retain` when the `committee_bits` of the two attestations intersect. fn update_covering_set( &mut self, - best_att: &AttestationRef<'a, E>, + best_att: &CompactAttestationRef<'a, E>, covered_validators: &HashMap, ) { if self.att.data.slot == best_att.data.slot && self.att.data.index == best_att.data.index { @@ -177,7 +177,7 @@ impl<'a, E: EthSpec> MaxCover for AttMaxCover<'a, E> { /// /// This isn't optimal, but with the Altair fork this code is obsolete and not worth upgrading. pub fn earliest_attestation_validators( - attestation: &AttestationRef, + attestation: &CompactAttestationRef, state: &BeaconState, base_state: &BeaconStateBase, ) -> BitList { diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index f06da2afb17..43b1c3abbb3 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -41,9 +41,8 @@ pub struct SplitAttestation { pub indexed: CompactIndexedAttestation, } -// TODO(electra): rename this type #[derive(Debug, Clone)] -pub struct AttestationRef<'a, E: EthSpec> { +pub struct CompactAttestationRef<'a, E: EthSpec> { pub checkpoint: &'a CheckpointKey, pub data: &'a CompactAttestationData, pub indexed: &'a CompactIndexedAttestation, @@ -97,8 +96,8 @@ impl SplitAttestation { } } - pub fn as_ref(&self) -> AttestationRef { - AttestationRef { + pub fn as_ref(&self) -> CompactAttestationRef { + CompactAttestationRef { checkpoint: &self.checkpoint, data: &self.data, indexed: &self.indexed, @@ -106,7 +105,7 @@ impl SplitAttestation { } } -impl<'a, E: EthSpec> AttestationRef<'a, E> { +impl<'a, E: EthSpec> CompactAttestationRef<'a, E> { pub fn attestation_data(&self) -> AttestationData { AttestationData { slot: self.data.slot, @@ -171,7 +170,7 @@ impl CompactIndexedAttestation { } } - pub fn aggregate(&mut self, other: &Self) { + pub fn aggregate(&mut self, other: &Self) -> Option<()> { match (self, other) { (CompactIndexedAttestation::Base(this), CompactIndexedAttestation::Base(other)) => { this.aggregate(other) @@ -181,7 +180,7 @@ impl CompactIndexedAttestation { CompactIndexedAttestation::Electra(other), ) => this.aggregate_same_committee(other), // TODO(electra) is a mix of electra and base compact indexed attestations an edge case we need to deal with? - _ => (), + _ => None, } } } @@ -193,7 +192,7 @@ impl CompactIndexedAttestationBase { .is_zero() } - pub fn aggregate(&mut self, other: &Self) { + pub fn aggregate(&mut self, other: &Self) -> Option<()> { self.attesting_indices = self .attesting_indices .drain(..) @@ -202,6 +201,8 @@ impl CompactIndexedAttestationBase { .collect(); self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); self.signature.add_assign_aggregate(&other.signature); + + Some(()) } } @@ -215,9 +216,11 @@ impl CompactIndexedAttestationElectra { .is_zero() } - pub fn aggregate_same_committee(&mut self, other: &Self) { + pub fn aggregate_same_committee(&mut self, other: &Self) -> Option<()> { // TODO(electra): remove assert in favour of Result - assert_eq!(self.committee_bits, other.committee_bits); + if self.committee_bits != other.committee_bits { + return None; + } self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); self.attesting_indices = self .attesting_indices @@ -226,34 +229,48 @@ impl CompactIndexedAttestationElectra { .dedup() .collect(); self.signature.add_assign_aggregate(&other.signature); + Some(()) } - pub fn aggregate_with_disjoint_committees(&mut self, other: &Self) { - // TODO(electra): remove asserts or use Result - assert!(self + pub fn aggregate_with_disjoint_committees(&mut self, other: &Self) -> Option<()> { + if !self .committee_bits .intersection(&other.committee_bits) - .is_zero(),); + .is_zero() + { + return None; + } // The attestation being aggregated in must only have 1 committee bit set. - assert_eq!(other.committee_bits.num_set_bits(), 1); + if other.committee_bits.num_set_bits() != 1 { + return None; + } + // Check we are aggregating in increasing committee index order (so we can append // aggregation bits). - assert!(self.committee_bits.highest_set_bit() < other.committee_bits.highest_set_bit()); + if self.committee_bits.highest_set_bit() >= other.committee_bits.highest_set_bit() { + return None; + } self.committee_bits = self.committee_bits.union(&other.committee_bits); - self.aggregation_bits = - bitlist_extend(&self.aggregation_bits, &other.aggregation_bits).unwrap(); - self.attesting_indices = self - .attesting_indices - .drain(..) - .merge(other.attesting_indices.iter().copied()) - .dedup() - .collect(); - self.signature.add_assign_aggregate(&other.signature); + if let Some(agg_bits) = bitlist_extend(&self.aggregation_bits, &other.aggregation_bits) { + self.aggregation_bits = agg_bits; + + self.attesting_indices = self + .attesting_indices + .drain(..) + .merge(other.attesting_indices.iter().copied()) + .dedup() + .collect(); + self.signature.add_assign_aggregate(&other.signature); + + return Some(()); + } + + None } - pub fn committee_index(&self) -> u64 { - *self.get_committee_indices().first().unwrap_or(&0u64) + pub fn committee_index(&self) -> Option { + self.get_committee_indices().first().copied() } pub fn get_committee_indices(&self) -> Vec { @@ -350,27 +367,28 @@ impl AttestationMap { continue; } }; - let committee_index = electra_attestation.committee_index(); - if let Some(existing_attestation) = - best_attestations_by_committee.get_mut(&committee_index) - { - // Search for the best (most aggregation bits) attestation for this committee - // index. - if electra_attestation.aggregation_bits.num_set_bits() - > existing_attestation.aggregation_bits.num_set_bits() + if let Some(committee_index) = electra_attestation.committee_index() { + if let Some(existing_attestation) = + best_attestations_by_committee.get_mut(&committee_index) { - // New attestation is better than the previously known one for this - // committee. Replace it. - std::mem::swap(existing_attestation, &mut electra_attestation); + // Search for the best (most aggregation bits) attestation for this committee + // index. + if electra_attestation.aggregation_bits.num_set_bits() + > existing_attestation.aggregation_bits.num_set_bits() + { + // New attestation is better than the previously known one for this + // committee. Replace it. + std::mem::swap(existing_attestation, &mut electra_attestation); + } + // Put the inferior attestation into the list of aggregated attestations + // without performing any cross-committee aggregation. + aggregated_attestations + .push(CompactIndexedAttestation::Electra(electra_attestation)); + } else { + // First attestation seen for this committee. Place it in the map + // provisionally. + best_attestations_by_committee.insert(committee_index, electra_attestation); } - // Put the inferior attestation into the list of aggregated attestations - // without performing any cross-committee aggregation. - aggregated_attestations - .push(CompactIndexedAttestation::Electra(electra_attestation)); - } else { - // First attestation seen for this committee. Place it in the map - // provisionally. - best_attestations_by_committee.insert(committee_index, electra_attestation); } } @@ -399,7 +417,7 @@ impl AttestationMap { pub fn get_attestations<'a>( &'a self, checkpoint_key: &'a CheckpointKey, - ) -> impl Iterator> + 'a { + ) -> impl Iterator> + 'a { self.checkpoint_map .get(checkpoint_key) .into_iter() @@ -407,7 +425,7 @@ impl AttestationMap { } /// Iterate all attestations in the map. - pub fn iter(&self) -> impl Iterator> { + pub fn iter(&self) -> impl Iterator> { self.checkpoint_map .iter() .flat_map(|(checkpoint_key, attestation_map)| attestation_map.iter(checkpoint_key)) @@ -438,9 +456,9 @@ impl AttestationDataMap { pub fn iter<'a>( &'a self, checkpoint_key: &'a CheckpointKey, - ) -> impl Iterator> + 'a { + ) -> impl Iterator> + 'a { self.attestations.iter().flat_map(|(data, vec_indexed)| { - vec_indexed.iter().map(|indexed| AttestationRef { + vec_indexed.iter().map(|indexed| CompactAttestationRef { checkpoint: checkpoint_key, data, indexed, diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index be9f4902892..c7659651dac 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -11,7 +11,7 @@ mod sync_aggregate_id; pub use crate::bls_to_execution_changes::ReceivedPreCapella; pub use attestation::{earliest_attestation_validators, AttMaxCover}; -pub use attestation_storage::{AttestationRef, SplitAttestation}; +pub use attestation_storage::{CompactAttestationRef, SplitAttestation}; pub use max_cover::MaxCover; pub use persistence::{ PersistedOperationPool, PersistedOperationPoolV15, PersistedOperationPoolV20, @@ -227,7 +227,7 @@ impl OperationPool { state: &'a BeaconState, reward_cache: &'a RewardCache, total_active_balance: u64, - validity_filter: impl FnMut(&AttestationRef<'a, E>) -> bool + Send, + validity_filter: impl FnMut(&CompactAttestationRef<'a, E>) -> bool + Send, spec: &'a ChainSpec, ) -> impl Iterator> + Send { all_attestations @@ -251,8 +251,8 @@ impl OperationPool { pub fn get_attestations( &self, state: &BeaconState, - prev_epoch_validity_filter: impl for<'a> FnMut(&AttestationRef<'a, E>) -> bool + Send, - curr_epoch_validity_filter: impl for<'a> FnMut(&AttestationRef<'a, E>) -> bool + Send, + prev_epoch_validity_filter: impl for<'a> FnMut(&CompactAttestationRef<'a, E>) -> bool + Send, + curr_epoch_validity_filter: impl for<'a> FnMut(&CompactAttestationRef<'a, E>) -> bool + Send, spec: &ChainSpec, ) -> Result>, OpPoolError> { let fork_name = state.fork_name_unchecked(); @@ -1281,9 +1281,7 @@ mod release_tests { for att in &best_attestations { match fork_name { ForkName::Electra => { - // TODO(electra) some attestations only have 2 or 3 agg bits set - // others have 5 - assert!(att.num_set_aggregation_bits() >= 2); + assert!(att.num_set_aggregation_bits() >= small_step_size); } _ => { assert!(att.num_set_aggregation_bits() >= big_step_size); diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index 3b05cfab1fd..d2935dbca45 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -435,7 +435,12 @@ impl ForkChoiceTest { let validator_committee_index = 0; let validator_index = *head .beacon_state - .get_beacon_committee(current_slot, attestation.data().index) + .get_beacon_committee( + current_slot, + attestation + .committee_index() + .expect("should get committee index"), + ) .expect("should get committees") .committee .get(validator_committee_index) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 33249c49417..a79604003ee 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -13,7 +13,6 @@ pub mod attesting_indices_base { ) -> Result, BlockOperationError> { let attesting_indices = get_attesting_indices::(committee, &attestation.aggregation_bits)?; - Ok(IndexedAttestation::Base(IndexedAttestationBase { attesting_indices: VariableList::new(attesting_indices)?, data: attestation.data.clone(), @@ -52,6 +51,100 @@ pub mod attesting_indices_electra { use safe_arith::SafeArith; use types::*; + // TODO(electra) remove duplicate code + // get_indexed_attestation is almost an exact duplicate + // the only differences are the invalid selection proof + // and aggregator not in committee checks + pub fn get_indexed_attestation_from_signed_aggregate( + committees: &[BeaconCommittee], + signed_aggregate: &SignedAggregateAndProofElectra, + spec: &ChainSpec, + ) -> Result, BeaconStateError> { + let mut output: HashSet = HashSet::new(); + + let committee_bits = &signed_aggregate.message.aggregate.committee_bits; + let aggregation_bits = &signed_aggregate.message.aggregate.aggregation_bits; + let aggregator_index = signed_aggregate.message.aggregator_index; + let attestation = &signed_aggregate.message.aggregate; + + let committee_indices = get_committee_indices::(committee_bits); + + let mut committee_offset = 0; + + let committees_map: HashMap = committees + .iter() + .map(|committee| (committee.index, committee)) + .collect(); + + let committee_count_per_slot = committees.len() as u64; + let mut participant_count = 0; + + // TODO(electra): + // Note: this clones the signature which is known to be a relatively slow operation. + // + // Future optimizations should remove this clone. + let selection_proof = + SelectionProof::from(signed_aggregate.message.selection_proof.clone()); + + for index in committee_indices { + if let Some(&beacon_committee) = committees_map.get(&index) { + if !selection_proof + .is_aggregator(beacon_committee.committee.len(), spec) + .map_err(BeaconStateError::ArithError)? + { + return Err(BeaconStateError::InvalidSelectionProof { aggregator_index }); + } + + if !beacon_committee + .committee + .contains(&(aggregator_index as usize)) + { + return Err(BeaconStateError::AggregatorNotInCommittee { aggregator_index }); + } + + // This check is new to the spec's `process_attestation` in Electra. + if index >= committee_count_per_slot { + return Err(BeaconStateError::InvalidCommitteeIndex(index)); + } + + participant_count.safe_add_assign(beacon_committee.committee.len() as u64)?; + let committee_attesters = beacon_committee + .committee + .iter() + .enumerate() + .filter_map(|(i, &index)| { + if let Ok(aggregation_bit_index) = committee_offset.safe_add(i) { + if aggregation_bits.get(aggregation_bit_index).unwrap_or(false) { + return Some(index as u64); + } + } + None + }) + .collect::>(); + + output.extend(committee_attesters); + + committee_offset.safe_add_assign(beacon_committee.committee.len())?; + } else { + return Err(Error::NoCommitteeFound(index)); + } + } + + // This check is new to the spec's `process_attestation` in Electra. + if participant_count as usize != aggregation_bits.len() { + return Err(Error::InvalidBitfield); + } + + let mut indices = output.into_iter().collect_vec(); + indices.sort_unstable(); + + Ok(IndexedAttestation::Electra(IndexedAttestationElectra { + attesting_indices: VariableList::new(indices)?, + data: attestation.data.clone(), + signature: attestation.signature.clone(), + })) + } + pub fn get_indexed_attestation( committees: &[BeaconCommittee], attestation: &AttestationElectra, @@ -155,7 +248,7 @@ pub mod attesting_indices_electra { Ok(indices) } - fn get_committee_indices( + pub fn get_committee_indices( committee_bits: &BitVector, ) -> Vec { committee_bits diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index a179583556f..8cf39abf437 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -326,6 +326,7 @@ where genesis_validators_root, ); + // TODO(electra), signing root isnt unique in the case of electra let message = indexed_attestation.data().signing_root(domain); Ok(SignatureSet::multiple_pubkeys(signature, pubkeys, message)) @@ -436,7 +437,6 @@ where let message = slot.signing_root(domain); let signature = signed_aggregate_and_proof.message().selection_proof(); let validator_index = signed_aggregate_and_proof.message().aggregator_index(); - Ok(SignatureSet::single_pubkey( signature, get_pubkey(validator_index as usize).ok_or(Error::ValidatorUnknown(validator_index))?, diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index a2c5140395b..aa0932587ad 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -21,6 +21,7 @@ pub enum Error { SszTypesError(ssz_types::Error), AlreadySigned(usize), SubnetCountIsZero(ArithError), + IncorrectStateVariant, } #[superstruct( @@ -42,7 +43,9 @@ pub enum Error { serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), ), - ref_attributes(derive(TreeHash), tree_hash(enum_behaviour = "transparent")) + ref_attributes(derive(TreeHash), tree_hash(enum_behaviour = "transparent")), + cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"), + partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant") )] #[derive( Debug, @@ -74,14 +77,14 @@ pub struct Attestation { // TODO(electra): think about how to handle fork variants here impl TestRandom for Attestation { fn random_for_test(rng: &mut impl RngCore) -> Self { - let aggregation_bits: BitList = BitList::random_for_test(rng); - // let committee_bits: BitList = BitList::random_for_test(rng); + let aggregation_bits = BitList::random_for_test(rng); let data = AttestationData::random_for_test(rng); let signature = AggregateSignature::random_for_test(rng); + let committee_bits = BitVector::random_for_test(rng); - Self::Base(AttestationBase { + Self::Electra(AttestationElectra { aggregation_bits, - // committee_bits, + committee_bits, data, signature, }) @@ -166,9 +169,9 @@ impl Attestation { } } - pub fn committee_index(&self) -> u64 { + pub fn committee_index(&self) -> Option { match self { - Attestation::Base(att) => att.data.index, + Attestation::Base(att) => Some(att.data.index), Attestation::Electra(att) => att.committee_index(), } } @@ -217,12 +220,31 @@ impl<'a, E: EthSpec> AttestationRef<'a, E> { } } - pub fn committee_index(&self) -> u64 { + pub fn committee_index(&self) -> Option { match self { - AttestationRef::Base(att) => att.data.index, + AttestationRef::Base(att) => Some(att.data.index), AttestationRef::Electra(att) => att.committee_index(), } } + + pub fn set_aggregation_bits(&self) -> Vec { + match self { + Self::Base(att) => att + .aggregation_bits + .iter() + .enumerate() + .filter(|(_i, bit)| *bit) + .map(|(i, _bit)| i) + .collect::>(), + Self::Electra(att) => att + .aggregation_bits + .iter() + .enumerate() + .filter(|(_i, bit)| *bit) + .map(|(i, _bit)| i) + .collect::>(), + } + } } impl AttestationElectra { @@ -236,8 +258,8 @@ impl AttestationElectra { .is_zero() } - pub fn committee_index(&self) -> u64 { - *self.get_committee_indices().first().unwrap_or(&0u64) + pub fn committee_index(&self) -> Option { + self.get_committee_indices().first().cloned() } pub fn get_committee_indices(&self) -> Vec { diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index d9c7a78537a..bc20f3aa7bb 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -161,6 +161,12 @@ pub enum Error { MerkleTreeError(merkle_proof::MerkleTreeError), NoCommitteeFound(CommitteeIndex), InvalidCommitteeIndex(CommitteeIndex), + InvalidSelectionProof { + aggregator_index: u64, + }, + AggregatorNotInCommittee { + aggregator_index: u64, + }, } /// Control whether an epoch-indexed field can be indexed at the next epoch or not. diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index f5b55f6d9b2..d282e2f259c 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -206,14 +206,17 @@ impl Decode for IndexedAttestation { } } +// TODO(electra): think about how to handle fork variants here impl TestRandom for IndexedAttestation { fn random_for_test(rng: &mut impl RngCore) -> Self { let attesting_indices = VariableList::random_for_test(rng); + // let committee_bits: BitList = BitList::random_for_test(rng); let data = AttestationData::random_for_test(rng); let signature = AggregateSignature::random_for_test(rng); Self::Base(IndexedAttestationBase { attesting_indices, + // committee_bits, data, signature, }) diff --git a/consensus/types/src/signed_aggregate_and_proof.rs b/consensus/types/src/signed_aggregate_and_proof.rs index 57a2ce5babe..ddf1dedb040 100644 --- a/consensus/types/src/signed_aggregate_and_proof.rs +++ b/consensus/types/src/signed_aggregate_and_proof.rs @@ -74,7 +74,6 @@ impl SignedAggregateAndProof { genesis_validators_root, spec, ); - let target_epoch = message.aggregate().data().slot.epoch(E::slots_per_epoch()); let domain = spec.get_domain( target_epoch, diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index ec447956749..b5b05e710b6 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -40,13 +40,15 @@ impl SubnetId { /// Compute the subnet for an attestation where each slot in the /// attestation epoch contains `committee_count_per_slot` committees. pub fn compute_subnet_for_attestation( - attestation: &AttestationRef, + attestation: AttestationRef, committee_count_per_slot: u64, spec: &ChainSpec, ) -> Result { + let committee_index = attestation.committee_index().ok_or(ArithError::Overflow)?; + Self::compute_subnet::( attestation.data().slot, - attestation.committee_index(), + committee_index, committee_count_per_slot, spec, ) diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 4213edc4392..404fe41249b 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -14,11 +14,11 @@ use std::ops::Deref; use std::sync::Arc; use tokio::time::{sleep, sleep_until, Duration, Instant}; use tree_hash::TreeHash; +use types::ForkName; use types::{ - attestation::AttestationBase, AggregateSignature, Attestation, AttestationData, BitList, - ChainSpec, CommitteeIndex, EthSpec, Slot, + attestation::AttestationBase, AggregateSignature, Attestation, AttestationData, + AttestationElectra, BitList, BitVector, ChainSpec, CommitteeIndex, EthSpec, Slot, }; -use types::{AttestationElectra, BitVector, ForkName}; /// Builds an `AttestationService`. pub struct AttestationServiceBuilder { @@ -643,7 +643,7 @@ impl AttestationService { "aggregator" => signed_aggregate_and_proof.message().aggregator_index(), "signatures" => attestation.num_set_aggregation_bits(), "head_block" => format!("{:?}", attestation.data().beacon_block_root), - "committee_index" => attestation.data().index, + "committee_index" => attestation.committee_index(), "slot" => attestation.data().slot.as_u64(), "type" => "aggregated", ); @@ -657,7 +657,7 @@ impl AttestationService { "Failed to publish attestation"; "error" => %e, "aggregator" => signed_aggregate_and_proof.message().aggregator_index(), - "committee_index" => attestation.data().index, + "committee_index" => attestation.committee_index(), "slot" => attestation.data().slot.as_u64(), "type" => "aggregated", ); From b61d244c0c1c9c1a7c56b4661f8d051cc8b1b656 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 30 May 2024 11:52:38 -0400 Subject: [PATCH 123/227] fix some todos (#5817) --- beacon_node/beacon_chain/src/beacon_chain.rs | 44 -------------------- consensus/types/src/attestation.rs | 41 ++++++++++++++---- consensus/types/src/attester_slashing.rs | 11 +++-- consensus/types/src/beacon_block.rs | 3 -- 4 files changed, 42 insertions(+), 57 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 1428848fbc8..bdeb92be395 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -5145,50 +5145,6 @@ impl BeaconChain { }, ); - // TODO(electra): figure out what should *actually* be done here when we have attestations / attester_slashings of the wrong type - match &state { - BeaconState::Base(_) - | BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) => { - if !attestations_electra.is_empty() { - error!( - self.log, - "Tried to produce block with attestations of the wrong type"; - "slot" => slot, - "attestations" => attestations_electra.len(), - ); - } - if !attester_slashings_electra.is_empty() { - error!( - self.log, - "Tried to produce block with attester slashings of the wrong type"; - "slot" => slot, - "attester_slashings" => attester_slashings_electra.len(), - ); - } - } - BeaconState::Electra(_) => { - if !attestations_base.is_empty() { - error!( - self.log, - "Tried to produce block with attestations of the wrong type"; - "slot" => slot, - "attestations" => attestations_base.len(), - ); - } - if !attester_slashings_base.is_empty() { - error!( - self.log, - "Tried to produce block with attester slashings of the wrong type"; - "slot" => slot, - "attester_slashings" => attester_slashings_base.len(), - ); - } - } - }; - let (inner_block, maybe_blobs_and_proofs, execution_payload_value) = match &state { BeaconState::Base(_) => ( BeaconBlock::Base(BeaconBlockBase { diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index aa0932587ad..3df14feadef 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -488,7 +488,7 @@ mod tests { // This test will only pass with `blst`, if we run these tests with another // BLS library in future we will have to make it generic. #[test] - fn size_of() { + fn size_of_base() { use std::mem::size_of; let aggregation_bits = @@ -501,16 +501,43 @@ mod tests { assert_eq!(signature, 288 + 16); let attestation_expected = aggregation_bits + attestation_data + signature; - // TODO(electra) since we've removed attestation aggregation for electra variant - // i've updated the attestation value expected from 488 544 - // assert_eq!(attestation_expected, 488); assert_eq!(attestation_expected, 488); assert_eq!( - size_of::>(), + size_of::>(), attestation_expected ); } - // TODO(electra): can we do this with both variants or should we? - ssz_and_tree_hash_tests!(AttestationBase); + #[test] + fn size_of_electra() { + use std::mem::size_of; + + let aggregation_bits = + size_of::::MaxValidatorsPerSlot>>(); + let attestation_data = size_of::(); + let committee_bits = + size_of::::MaxCommitteesPerSlot>>(); + let signature = size_of::(); + + assert_eq!(aggregation_bits, 56); + assert_eq!(committee_bits, 56); + assert_eq!(attestation_data, 128); + assert_eq!(signature, 288 + 16); + + let attestation_expected = aggregation_bits + committee_bits + attestation_data + signature; + assert_eq!(attestation_expected, 544); + assert_eq!( + size_of::>(), + attestation_expected + ); + } + + mod base { + use super::*; + ssz_and_tree_hash_tests!(AttestationBase); + } + mod electra { + use super::*; + ssz_and_tree_hash_tests!(AttestationElectra); + } } diff --git a/consensus/types/src/attester_slashing.rs b/consensus/types/src/attester_slashing.rs index 6668f809b82..a8d4e6989c8 100644 --- a/consensus/types/src/attester_slashing.rs +++ b/consensus/types/src/attester_slashing.rs @@ -164,7 +164,12 @@ impl AttesterSlashing { mod tests { use super::*; use crate::*; - - // TODO(electra): should this be done for both variants? - ssz_and_tree_hash_tests!(AttesterSlashingBase); + mod base { + use super::*; + ssz_and_tree_hash_tests!(AttesterSlashingBase); + } + mod electra { + use super::*; + ssz_and_tree_hash_tests!(AttesterSlashingElectra); + } } diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 1052b20146d..f67a965955c 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -607,14 +607,12 @@ impl> BeaconBlockElectra /// Return a Electra block where the block has maximum size. pub fn full(spec: &ChainSpec) -> Self { let base_block: BeaconBlockBase<_, Payload> = BeaconBlockBase::full(spec); - // TODO(electra): check this let indexed_attestation: IndexedAttestationElectra = IndexedAttestationElectra { attesting_indices: VariableList::new(vec![0_u64; E::MaxValidatorsPerSlot::to_usize()]) .unwrap(), data: AttestationData::default(), signature: AggregateSignature::empty(), }; - // TODO(electra): fix this so we calculate this size correctly let attester_slashings = vec![ AttesterSlashingElectra { attestation_1: indexed_attestation.clone(), @@ -627,7 +625,6 @@ impl> BeaconBlockElectra aggregation_bits: BitList::with_capacity(E::MaxValidatorsPerSlot::to_usize()).unwrap(), data: AttestationData::default(), signature: AggregateSignature::empty(), - // TODO(electra): does this actually allocate the size correctly? committee_bits: BitVector::new(), }; let mut attestations_electra = vec![]; From 29ed1c5c2670cc854f4068de2a77364eedb4ba80 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 12 May 2024 08:17:14 -0400 Subject: [PATCH 124/227] add consolidations to merkle calc for inclusion proof --- consensus/types/src/beacon_block_body.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index fba542450de..0466d1b7680 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -251,6 +251,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, body.execution_payload.tree_hash_root(), body.bls_to_execution_changes.tree_hash_root(), body.blob_kzg_commitments.tree_hash_root(), + body.consolidations.tree_hash_root(), ]; let beacon_block_body_depth = leaves.len().next_power_of_two().ilog2() as usize; let tree = MerkleTree::create(&leaves, beacon_block_body_depth); From f9d354539a79ed1570a236360973405dbbd158c6 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Sat, 1 Jun 2024 18:51:00 +0200 Subject: [PATCH 125/227] Remove Duplicate KZG Commitment Merkle Proof Code (#5874) * Remove Duplicate KZG Commitment Merkle Proof Code * s/tree_lists/fields/ --- consensus/types/src/beacon_block_body.rs | 211 +++++++++-------------- 1 file changed, 85 insertions(+), 126 deletions(-) diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index 0466d1b7680..76a2fc1872e 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -2,6 +2,7 @@ use crate::test_utils::TestRandom; use crate::*; use derivative::Derivative; use merkle_proof::{MerkleTree, MerkleTreeError}; +use metastruct::metastruct; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use std::marker::PhantomData; @@ -50,6 +51,14 @@ pub const BLOB_KZG_COMMITMENTS_INDEX: usize = 11; ), arbitrary(bound = "E: EthSpec, Payload: AbstractExecPayload"), ), + specific_variant_attributes( + Base(metastruct(mappings(beacon_block_body_base_fields(groups(fields))))), + Altair(metastruct(mappings(beacon_block_body_altair_fields(groups(fields))))), + Bellatrix(metastruct(mappings(beacon_block_body_bellatrix_fields(groups(fields))))), + Capella(metastruct(mappings(beacon_block_body_capella_fields(groups(fields))))), + Deneb(metastruct(mappings(beacon_block_body_deneb_fields(groups(fields))))), + Electra(metastruct(mappings(beacon_block_body_electra_fields(groups(fields))))), + ), cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"), partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant") )] @@ -108,6 +117,7 @@ pub struct BeaconBlockBody = FullPay #[superstruct(only(Electra))] pub consolidations: VariableList, #[superstruct(only(Base, Altair))] + #[metastruct(exclude_from(fields))] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] #[serde(skip)] @@ -132,139 +142,88 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, } } - /// Produces the proof of inclusion for a `KzgCommitment` in `self.blob_kzg_commitments` - /// at `index`. - pub fn kzg_commitment_merkle_proof( - &self, - index: usize, - ) -> Result, Error> { + fn body_merkle_leaves(&self) -> Vec { + let mut leaves = vec![]; match self { - Self::Base(_) | Self::Altair(_) | Self::Bellatrix(_) | Self::Capella(_) => { - Err(Error::IncorrectStateVariant) + Self::Base(body) => { + beacon_block_body_base_fields!(body, |_, field| leaves + .push(field.tree_hash_root())); + } + Self::Altair(body) => { + beacon_block_body_altair_fields!(body, |_, field| leaves + .push(field.tree_hash_root())); + } + Self::Bellatrix(body) => { + beacon_block_body_bellatrix_fields!(body, |_, field| leaves + .push(field.tree_hash_root())); + } + Self::Capella(body) => { + beacon_block_body_capella_fields!(body, |_, field| leaves + .push(field.tree_hash_root())); } Self::Deneb(body) => { - // We compute the branches by generating 2 merkle trees: - // 1. Merkle tree for the `blob_kzg_commitments` List object - // 2. Merkle tree for the `BeaconBlockBody` container - // We then merge the branches for both the trees all the way up to the root. - - // Part1 (Branches for the subtree rooted at `blob_kzg_commitments`) - // - // Branches for `blob_kzg_commitments` without length mix-in - let depth = E::max_blob_commitments_per_block() - .next_power_of_two() - .ilog2(); - let leaves: Vec<_> = body - .blob_kzg_commitments - .iter() - .map(|commitment| commitment.tree_hash_root()) - .collect(); - let tree = MerkleTree::create(&leaves, depth as usize); - let (_, mut proof) = tree - .generate_proof(index, depth as usize) - .map_err(Error::MerkleTreeError)?; - - // Add the branch corresponding to the length mix-in. - let length = body.blob_kzg_commitments.len(); - let usize_len = std::mem::size_of::(); - let mut length_bytes = [0; BYTES_PER_CHUNK]; - length_bytes - .get_mut(0..usize_len) - .ok_or(Error::MerkleTreeError(MerkleTreeError::PleaseNotifyTheDevs))? - .copy_from_slice(&length.to_le_bytes()); - let length_root = Hash256::from_slice(length_bytes.as_slice()); - proof.push(length_root); - - // Part 2 - // Branches for `BeaconBlockBody` container - let leaves = [ - body.randao_reveal.tree_hash_root(), - body.eth1_data.tree_hash_root(), - body.graffiti.tree_hash_root(), - body.proposer_slashings.tree_hash_root(), - body.attester_slashings.tree_hash_root(), - body.attestations.tree_hash_root(), - body.deposits.tree_hash_root(), - body.voluntary_exits.tree_hash_root(), - body.sync_aggregate.tree_hash_root(), - body.execution_payload.tree_hash_root(), - body.bls_to_execution_changes.tree_hash_root(), - body.blob_kzg_commitments.tree_hash_root(), - ]; - let beacon_block_body_depth = leaves.len().next_power_of_two().ilog2() as usize; - let tree = MerkleTree::create(&leaves, beacon_block_body_depth); - let (_, mut proof_body) = tree - .generate_proof(BLOB_KZG_COMMITMENTS_INDEX, beacon_block_body_depth) - .map_err(Error::MerkleTreeError)?; - // Join the proofs for the subtree and the main tree - proof.append(&mut proof_body); - - debug_assert_eq!(proof.len(), E::kzg_proof_inclusion_proof_depth()); - Ok(proof.into()) + beacon_block_body_deneb_fields!(body, |_, field| leaves + .push(field.tree_hash_root())); } - // TODO(electra): De-duplicate proof computation. Self::Electra(body) => { - // We compute the branches by generating 2 merkle trees: - // 1. Merkle tree for the `blob_kzg_commitments` List object - // 2. Merkle tree for the `BeaconBlockBody` container - // We then merge the branches for both the trees all the way up to the root. - - // Part1 (Branches for the subtree rooted at `blob_kzg_commitments`) - // - // Branches for `blob_kzg_commitments` without length mix-in - let depth = E::max_blob_commitments_per_block() - .next_power_of_two() - .ilog2(); - let leaves: Vec<_> = body - .blob_kzg_commitments - .iter() - .map(|commitment| commitment.tree_hash_root()) - .collect(); - let tree = MerkleTree::create(&leaves, depth as usize); - let (_, mut proof) = tree - .generate_proof(index, depth as usize) - .map_err(Error::MerkleTreeError)?; - - // Add the branch corresponding to the length mix-in. - let length = body.blob_kzg_commitments.len(); - let usize_len = std::mem::size_of::(); - let mut length_bytes = [0; BYTES_PER_CHUNK]; - length_bytes - .get_mut(0..usize_len) - .ok_or(Error::MerkleTreeError(MerkleTreeError::PleaseNotifyTheDevs))? - .copy_from_slice(&length.to_le_bytes()); - let length_root = Hash256::from_slice(length_bytes.as_slice()); - proof.push(length_root); - - // Part 2 - // Branches for `BeaconBlockBody` container - let leaves = [ - body.randao_reveal.tree_hash_root(), - body.eth1_data.tree_hash_root(), - body.graffiti.tree_hash_root(), - body.proposer_slashings.tree_hash_root(), - body.attester_slashings.tree_hash_root(), - body.attestations.tree_hash_root(), - body.deposits.tree_hash_root(), - body.voluntary_exits.tree_hash_root(), - body.sync_aggregate.tree_hash_root(), - body.execution_payload.tree_hash_root(), - body.bls_to_execution_changes.tree_hash_root(), - body.blob_kzg_commitments.tree_hash_root(), - body.consolidations.tree_hash_root(), - ]; - let beacon_block_body_depth = leaves.len().next_power_of_two().ilog2() as usize; - let tree = MerkleTree::create(&leaves, beacon_block_body_depth); - let (_, mut proof_body) = tree - .generate_proof(BLOB_KZG_COMMITMENTS_INDEX, beacon_block_body_depth) - .map_err(Error::MerkleTreeError)?; - // Join the proofs for the subtree and the main tree - proof.append(&mut proof_body); - - debug_assert_eq!(proof.len(), E::kzg_proof_inclusion_proof_depth()); - Ok(proof.into()) + beacon_block_body_electra_fields!(body, |_, field| leaves + .push(field.tree_hash_root())); } } + leaves + } + + /// Produces the proof of inclusion for a `KzgCommitment` in `self.blob_kzg_commitments` + /// at `index`. + pub fn kzg_commitment_merkle_proof( + &self, + index: usize, + ) -> Result, Error> { + // We compute the branches by generating 2 merkle trees: + // 1. Merkle tree for the `blob_kzg_commitments` List object + // 2. Merkle tree for the `BeaconBlockBody` container + // We then merge the branches for both the trees all the way up to the root. + + // Part1 (Branches for the subtree rooted at `blob_kzg_commitments`) + // + // Branches for `blob_kzg_commitments` without length mix-in + let blob_leaves = self + .blob_kzg_commitments()? + .iter() + .map(|commitment| commitment.tree_hash_root()) + .collect::>(); + let depth = E::max_blob_commitments_per_block() + .next_power_of_two() + .ilog2(); + let tree = MerkleTree::create(&blob_leaves, depth as usize); + let (_, mut proof) = tree + .generate_proof(index, depth as usize) + .map_err(Error::MerkleTreeError)?; + + // Add the branch corresponding to the length mix-in. + let length = blob_leaves.len(); + let usize_len = std::mem::size_of::(); + let mut length_bytes = [0; BYTES_PER_CHUNK]; + length_bytes + .get_mut(0..usize_len) + .ok_or(Error::MerkleTreeError(MerkleTreeError::PleaseNotifyTheDevs))? + .copy_from_slice(&length.to_le_bytes()); + let length_root = Hash256::from_slice(length_bytes.as_slice()); + proof.push(length_root); + + // Part 2 + // Branches for `BeaconBlockBody` container + let body_leaves = self.body_merkle_leaves(); + let beacon_block_body_depth = body_leaves.len().next_power_of_two().ilog2() as usize; + let tree = MerkleTree::create(&body_leaves, beacon_block_body_depth); + let (_, mut proof_body) = tree + .generate_proof(BLOB_KZG_COMMITMENTS_INDEX, beacon_block_body_depth) + .map_err(Error::MerkleTreeError)?; + // Join the proofs for the subtree and the main tree + proof.append(&mut proof_body); + debug_assert_eq!(proof.len(), E::kzg_proof_inclusion_proof_depth()); + + Ok(proof.into()) } /// Return `true` if this block body has a non-zero number of blobs. From b21b1086f1991b5792bbcfb5fe14181a1ab38a18 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 13 Jun 2024 16:40:52 -0400 Subject: [PATCH 126/227] fix compile --- beacon_node/beacon_chain/src/attestation_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 2b52f69cc67..d82685c48c0 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -428,7 +428,7 @@ fn process_slash_info( let (indexed_attestation, check_signature, err) = match slash_info { SignatureNotChecked(attestation, err) => { if let Error::UnknownHeadBlock { .. } = err { - if attestation.data.beacon_block_root == attestation.data.target.root { + if attestation.data().beacon_block_root == attestation.data().target.root { return err; } } From 35e07eb0a917d0b067e947b5421cdb9d6f15b2da Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Thu, 13 Jun 2024 19:27:36 -0700 Subject: [PATCH 127/227] Fix slasher tests (#5906) * Fix electra tests * Add electra attestations to double vote tests --- slasher/src/lib.rs | 69 ++++----- slasher/src/test_utils.rs | 13 +- slasher/tests/attester_slashings.rs | 228 ++++++++++++++++++++-------- 3 files changed, 200 insertions(+), 110 deletions(-) diff --git a/slasher/src/lib.rs b/slasher/src/lib.rs index 339ca1f7d3a..0c097ac77fb 100644 --- a/slasher/src/lib.rs +++ b/slasher/src/lib.rs @@ -60,49 +60,46 @@ impl AttesterSlashingStatus { Ok(match self { NotSlashable => None, AlreadyDoubleVoted => None, - DoubleVote(existing) | SurroundedByExisting(existing) => match *existing { - IndexedAttestation::Base(existing_att) => { - Some(AttesterSlashing::Base(AttesterSlashingBase { - attestation_1: existing_att, - attestation_2: new_attestation - .as_base() - .map_err(|e| format!("{e:?}"))? - .clone(), - })) - } - IndexedAttestation::Electra(existing_att) => { - Some(AttesterSlashing::Electra(AttesterSlashingElectra { - attestation_1: existing_att, - // A double vote should never convert, a surround vote where the surrounding - // vote is electra may convert. + DoubleVote(existing) | SurroundedByExisting(existing) => { + match (&*existing, new_attestation) { + (IndexedAttestation::Base(existing_att), IndexedAttestation::Base(new)) => { + Some(AttesterSlashing::Base(AttesterSlashingBase { + attestation_1: existing_att.clone(), + attestation_2: new.clone(), + })) + } + // A slashing involving an electra attestation type must return an `AttesterSlashingElectra` type + (_, _) => Some(AttesterSlashing::Electra(AttesterSlashingElectra { + attestation_1: existing + .clone() + .to_electra() + .map_err(|e| format!("{e:?}"))?, attestation_2: new_attestation .clone() .to_electra() .map_err(|e| format!("{e:?}"))?, + })), + } + } + SurroundsExisting(existing) => match (&*existing, new_attestation) { + (IndexedAttestation::Base(existing_att), IndexedAttestation::Base(new)) => { + Some(AttesterSlashing::Base(AttesterSlashingBase { + attestation_1: new.clone(), + attestation_2: existing_att.clone(), })) } + // A slashing involving an electra attestation type must return an `AttesterSlashingElectra` type + (_, _) => Some(AttesterSlashing::Electra(AttesterSlashingElectra { + attestation_1: new_attestation + .clone() + .to_electra() + .map_err(|e| format!("{e:?}"))?, + attestation_2: existing + .clone() + .to_electra() + .map_err(|e| format!("{e:?}"))?, + })), }, - SurroundsExisting(existing) => { - match new_attestation { - IndexedAttestation::Base(new_attestation) => { - Some(AttesterSlashing::Base(AttesterSlashingBase { - attestation_1: existing - .as_base() - .map_err(|e| format!("{e:?}"))? - .clone(), - attestation_2: new_attestation.clone(), - })) - } - IndexedAttestation::Electra(new_attestation) => { - Some(AttesterSlashing::Electra(AttesterSlashingElectra { - attestation_1: existing.to_electra().map_err(|e| format!("{e:?}"))?, - // A double vote should never convert, a surround vote where the surrounding - // vote is electra may convert. - attestation_2: new_attestation.clone(), - })) - } - } - } }) } } diff --git a/slasher/src/test_utils.rs b/slasher/src/test_utils.rs index 8bbf042c2dd..f22a680bb7e 100644 --- a/slasher/src/test_utils.rs +++ b/slasher/src/test_utils.rs @@ -63,7 +63,6 @@ pub fn att_slashing( attestation_1: &IndexedAttestation, attestation_2: &IndexedAttestation, ) -> AttesterSlashing { - // TODO(electra): fix this one we superstruct IndexedAttestation (return the correct type) match (attestation_1, attestation_2) { (IndexedAttestation::Base(att1), IndexedAttestation::Base(att2)) => { AttesterSlashing::Base(AttesterSlashingBase { @@ -71,13 +70,11 @@ pub fn att_slashing( attestation_2: att2.clone(), }) } - (IndexedAttestation::Electra(att1), IndexedAttestation::Electra(att2)) => { - AttesterSlashing::Electra(AttesterSlashingElectra { - attestation_1: att1.clone(), - attestation_2: att2.clone(), - }) - } - _ => panic!("attestations must be of the same type"), + // A slashing involving an electra attestation type must return an electra AttesterSlashing type + (_, _) => AttesterSlashing::Electra(AttesterSlashingElectra { + attestation_1: attestation_1.clone().to_electra().unwrap(), + attestation_2: attestation_2.clone().to_electra().unwrap(), + }), } } diff --git a/slasher/tests/attester_slashings.rs b/slasher/tests/attester_slashings.rs index 40d9fa511c6..b74e491e4b3 100644 --- a/slasher/tests/attester_slashings.rs +++ b/slasher/tests/attester_slashings.rs @@ -5,7 +5,9 @@ use maplit::hashset; use rayon::prelude::*; use slasher::{ config::DEFAULT_CHUNK_SIZE, - test_utils::{att_slashing, indexed_att, slashed_validators_from_slashings, E}, + test_utils::{ + att_slashing, indexed_att, indexed_att_electra, slashed_validators_from_slashings, E, + }, Config, Slasher, }; use std::collections::HashSet; @@ -15,23 +17,35 @@ use types::{AttesterSlashing, Epoch, IndexedAttestation}; #[test] fn double_vote_single_val() { let v = vec![99]; - let att1 = indexed_att(&v, 0, 1, 0); - let att2 = indexed_att(&v, 0, 1, 1); - let slashings = hashset![att_slashing(&att1, &att2)]; - let attestations = vec![att1, att2]; - slasher_test_indiv(&attestations, &slashings, 1); - slasher_test_indiv(&attestations, &slashings, 1000); + for (att1, att2) in [ + (indexed_att(&v, 0, 1, 0), indexed_att(&v, 0, 1, 1)), + ( + indexed_att_electra(&v, 0, 1, 0), + indexed_att_electra(&v, 0, 1, 1), + ), + ] { + let slashings = hashset![att_slashing(&att1, &att2)]; + let attestations = vec![att1, att2]; + slasher_test_indiv(&attestations, &slashings, 1); + slasher_test_indiv(&attestations, &slashings, 1000); + } } #[test] fn double_vote_multi_vals() { let v = vec![0, 1, 2]; - let att1 = indexed_att(&v, 0, 1, 0); - let att2 = indexed_att(&v, 0, 1, 1); - let slashings = hashset![att_slashing(&att1, &att2)]; - let attestations = vec![att1, att2]; - slasher_test_indiv(&attestations, &slashings, 1); - slasher_test_indiv(&attestations, &slashings, 1000); + for (att1, att2) in [ + (indexed_att(&v, 0, 1, 0), indexed_att(&v, 0, 1, 1)), + ( + indexed_att_electra(&v, 0, 1, 0), + indexed_att_electra(&v, 0, 1, 1), + ), + ] { + let slashings = hashset![att_slashing(&att1, &att2)]; + let attestations = vec![att1, att2]; + slasher_test_indiv(&attestations, &slashings, 1); + slasher_test_indiv(&attestations, &slashings, 1000); + } } // A subset of validators double vote. @@ -39,12 +53,18 @@ fn double_vote_multi_vals() { fn double_vote_some_vals() { let v1 = vec![0, 1, 2, 3, 4, 5, 6]; let v2 = vec![0, 2, 4, 6]; - let att1 = indexed_att(v1, 0, 1, 0); - let att2 = indexed_att(v2, 0, 1, 1); - let slashings = hashset![att_slashing(&att1, &att2)]; - let attestations = vec![att1, att2]; - slasher_test_indiv(&attestations, &slashings, 1); - slasher_test_indiv(&attestations, &slashings, 1000); + for (att1, att2) in [ + (indexed_att(&v1, 0, 1, 0), indexed_att(&v2, 0, 1, 1)), + ( + indexed_att_electra(&v1, 0, 1, 0), + indexed_att_electra(&v2, 0, 1, 1), + ), + ] { + let slashings = hashset![att_slashing(&att1, &att2)]; + let attestations = vec![att1, att2]; + slasher_test_indiv(&attestations, &slashings, 1); + slasher_test_indiv(&attestations, &slashings, 1000); + } } // A subset of validators double vote, others vote twice for the same thing. @@ -53,13 +73,23 @@ fn double_vote_some_vals_repeat() { let v1 = vec![0, 1, 2, 3, 4, 5, 6]; let v2 = vec![0, 2, 4, 6]; let v3 = vec![1, 3, 5]; - let att1 = indexed_att(v1, 0, 1, 0); - let att2 = indexed_att(v2, 0, 1, 1); - let att3 = indexed_att(v3, 0, 1, 0); - let slashings = hashset![att_slashing(&att1, &att2)]; - let attestations = vec![att1, att2, att3]; - slasher_test_indiv(&attestations, &slashings, 1); - slasher_test_indiv(&attestations, &slashings, 1000); + for (att1, att2, att3) in [ + ( + indexed_att(&v1, 0, 1, 0), + indexed_att(&v2, 0, 1, 1), + indexed_att(&v3, 0, 1, 0), + ), + ( + indexed_att_electra(&v1, 0, 1, 0), + indexed_att_electra(&v2, 0, 1, 1), + indexed_att_electra(&v3, 0, 1, 0), + ), + ] { + let slashings = hashset![att_slashing(&att1, &att2)]; + let attestations = vec![att1, att2, att3]; + slasher_test_indiv(&attestations, &slashings, 1); + slasher_test_indiv(&attestations, &slashings, 1000); + } } // Nobody double votes, nobody gets slashed. @@ -67,11 +97,17 @@ fn double_vote_some_vals_repeat() { fn no_double_vote_same_target() { let v1 = vec![0, 1, 2, 3, 4, 5, 6]; let v2 = vec![0, 1, 2, 3, 4, 5, 7, 8]; - let att1 = indexed_att(v1, 0, 1, 0); - let att2 = indexed_att(v2, 0, 1, 0); - let attestations = vec![att1, att2]; - slasher_test_indiv(&attestations, &hashset! {}, 1); - slasher_test_indiv(&attestations, &hashset! {}, 1000); + for (att1, att2) in [ + (indexed_att(&v1, 0, 1, 0), indexed_att(&v2, 0, 1, 0)), + ( + indexed_att_electra(&v1, 0, 1, 0), + indexed_att_electra(&v2, 0, 1, 0), + ), + ] { + let attestations = vec![att1, att2]; + slasher_test_indiv(&attestations, &hashset! {}, 1); + slasher_test_indiv(&attestations, &hashset! {}, 1000); + } } // Two groups votes for different things, no slashings. @@ -79,73 +115,133 @@ fn no_double_vote_same_target() { fn no_double_vote_distinct_vals() { let v1 = vec![0, 1, 2, 3]; let v2 = vec![4, 5, 6, 7]; - let att1 = indexed_att(v1, 0, 1, 0); - let att2 = indexed_att(v2, 0, 1, 1); - let attestations = vec![att1, att2]; - slasher_test_indiv(&attestations, &hashset! {}, 1); - slasher_test_indiv(&attestations, &hashset! {}, 1000); + for (att1, att2) in [ + (indexed_att(&v1, 0, 1, 0), indexed_att(&v2, 0, 1, 0)), + ( + indexed_att_electra(&v1, 0, 1, 0), + indexed_att_electra(&v2, 0, 1, 1), + ), + ] { + let attestations = vec![att1, att2]; + slasher_test_indiv(&attestations, &hashset! {}, 1); + slasher_test_indiv(&attestations, &hashset! {}, 1000); + } } #[test] fn no_double_vote_repeated() { let v = vec![0, 1, 2, 3, 4]; - let att1 = indexed_att(v, 0, 1, 0); - let att2 = att1.clone(); - let attestations = vec![att1, att2]; - slasher_test_indiv(&attestations, &hashset! {}, 1); - slasher_test_batch(&attestations, &hashset! {}, 1); - parallel_slasher_test(&attestations, hashset! {}, 1); + for att1 in [indexed_att(&v, 0, 1, 0), indexed_att_electra(&v, 0, 1, 0)] { + let att2 = att1.clone(); + let attestations = vec![att1, att2]; + slasher_test_indiv(&attestations, &hashset! {}, 1); + slasher_test_batch(&attestations, &hashset! {}, 1); + parallel_slasher_test(&attestations, hashset! {}, 1); + } } #[test] fn surrounds_existing_single_val_single_chunk() { let v = vec![0]; - let att1 = indexed_att(&v, 1, 2, 0); - let att2 = indexed_att(&v, 0, 3, 0); - let slashings = hashset![att_slashing(&att2, &att1)]; - slasher_test_indiv(&[att1, att2], &slashings, 3); + for (att1, att2) in [ + (indexed_att(&v, 1, 2, 0), indexed_att(&v, 0, 3, 0)), + (indexed_att(&v, 1, 2, 0), indexed_att_electra(&v, 0, 3, 0)), + ( + indexed_att_electra(&v, 1, 2, 0), + indexed_att_electra(&v, 0, 3, 0), + ), + ] { + let slashings = hashset![att_slashing(&att2, &att1)]; + slasher_test_indiv(&[att1, att2], &slashings, 3); + } } #[test] fn surrounds_existing_multi_vals_single_chunk() { let validators = vec![0, 16, 1024, 300_000, 300_001]; - let att1 = indexed_att(validators.clone(), 1, 2, 0); - let att2 = indexed_att(validators, 0, 3, 0); - let slashings = hashset![att_slashing(&att2, &att1)]; - slasher_test_indiv(&[att1, att2], &slashings, 3); + for (att1, att2) in [ + ( + indexed_att(&validators, 1, 2, 0), + indexed_att(&validators, 0, 3, 0), + ), + ( + indexed_att(&validators, 1, 2, 0), + indexed_att_electra(&validators, 0, 3, 0), + ), + ( + indexed_att_electra(&validators, 1, 2, 0), + indexed_att_electra(&validators, 0, 3, 0), + ), + ] { + let slashings = hashset![att_slashing(&att2, &att1)]; + slasher_test_indiv(&[att1, att2], &slashings, 3); + } } #[test] fn surrounds_existing_many_chunks() { let v = vec![0]; let chunk_size = DEFAULT_CHUNK_SIZE as u64; - let att1 = indexed_att(&v, 3 * chunk_size, 3 * chunk_size + 1, 0); - let att2 = indexed_att(&v, 0, 3 * chunk_size + 2, 0); - let slashings = hashset![att_slashing(&att2, &att1)]; - let attestations = vec![att1, att2]; - slasher_test_indiv(&attestations, &slashings, 4 * chunk_size); + for (att1, att2) in [ + ( + indexed_att(&v, 3 * chunk_size, 3 * chunk_size + 1, 0), + indexed_att(&v, 0, 3 * chunk_size + 2, 0), + ), + ( + indexed_att(&v, 3 * chunk_size, 3 * chunk_size + 1, 0), + indexed_att_electra(&v, 0, 3 * chunk_size + 2, 0), + ), + ( + indexed_att_electra(&v, 3 * chunk_size, 3 * chunk_size + 1, 0), + indexed_att_electra(&v, 0, 3 * chunk_size + 2, 0), + ), + ] { + let slashings = hashset![att_slashing(&att2, &att1)]; + let attestations = vec![att1, att2]; + slasher_test_indiv(&attestations, &slashings, 4 * chunk_size); + } } #[test] fn surrounded_by_single_val_single_chunk() { let v = vec![0]; - let att1 = indexed_att(&v, 0, 15, 0); - let att2 = indexed_att(&v, 1, 14, 0); - let slashings = hashset![att_slashing(&att1, &att2)]; - let attestations = vec![att1, att2]; - slasher_test_indiv(&attestations, &slashings, 15); + for (att1, att2) in [ + (indexed_att(&v, 0, 15, 0), indexed_att(&v, 1, 14, 0)), + (indexed_att(&v, 0, 15, 0), indexed_att_electra(&v, 1, 14, 0)), + ( + indexed_att_electra(&v, 0, 15, 0), + indexed_att_electra(&v, 1, 14, 0), + ), + ] { + let slashings = hashset![att_slashing(&att1, &att2)]; + let attestations = vec![att1, att2]; + slasher_test_indiv(&attestations, &slashings, 15); + } } #[test] fn surrounded_by_single_val_multi_chunk() { let v = vec![0]; let chunk_size = DEFAULT_CHUNK_SIZE as u64; - let att1 = indexed_att(&v, 0, 3 * chunk_size, 0); - let att2 = indexed_att(&v, chunk_size, chunk_size + 1, 0); - let slashings = hashset![att_slashing(&att1, &att2)]; - let attestations = vec![att1, att2]; - slasher_test_indiv(&attestations, &slashings, 3 * chunk_size); - slasher_test_indiv(&attestations, &slashings, 4 * chunk_size); + for (att1, att2) in [ + ( + indexed_att(&v, 0, 3 * chunk_size, 0), + indexed_att(&v, chunk_size, chunk_size + 1, 0), + ), + ( + indexed_att(&v, 0, 3 * chunk_size, 0), + indexed_att_electra(&v, chunk_size, chunk_size + 1, 0), + ), + ( + indexed_att_electra(&v, 0, 3 * chunk_size, 0), + indexed_att_electra(&v, chunk_size, chunk_size + 1, 0), + ), + ] { + let slashings = hashset![att_slashing(&att1, &att2)]; + let attestations = vec![att1, att2]; + slasher_test_indiv(&attestations, &slashings, 3 * chunk_size); + slasher_test_indiv(&attestations, &slashings, 4 * chunk_size); + } } // Process each attestation individually, and confirm that the slashings produced are as expected. From d7f3c9583e455ee5d586c6104dc8f217981ed887 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 14 Jun 2024 12:32:20 +1000 Subject: [PATCH 128/227] Update superstruct to 0.8 --- Cargo.lock | 3 ++- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5c165cec899..3d6fddc0353 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8041,7 +8041,8 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "superstruct" version = "0.8.0" -source = "git+https://github.com/sigp/superstruct?rev=45eecbfb9708c9fe11dbb6a6a5bd8d618f02269e#45eecbfb9708c9fe11dbb6a6a5bd8d618f02269e" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0f31f730ad9e579364950e10d6172b4a9bd04b447edf5988b066a860cc340e" dependencies = [ "darling", "itertools", diff --git a/Cargo.toml b/Cargo.toml index 1059c746253..d67f6edf1dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -162,7 +162,7 @@ smallvec = "1.11.2" snap = "1" ssz_types = "0.6" strum = { version = "0.24", features = ["derive"] } -superstruct = { git = "https://github.com/sigp/superstruct", rev = "45eecbfb9708c9fe11dbb6a6a5bd8d618f02269e" } +superstruct = "0.8" syn = "1" sysinfo = "0.26" tempfile = "3" From c4f2284dbe00ade38e93b1bac35c235386759cb9 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 14 Jun 2024 12:50:18 +1000 Subject: [PATCH 129/227] Small cleanup in slasher tests --- slasher/src/test_utils.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/slasher/src/test_utils.rs b/slasher/src/test_utils.rs index f22a680bb7e..ef623911bfa 100644 --- a/slasher/src/test_utils.rs +++ b/slasher/src/test_utils.rs @@ -39,7 +39,6 @@ pub fn indexed_att( target_epoch: u64, target_root: u64, ) -> IndexedAttestation { - // TODO(electra) make fork-agnostic IndexedAttestation::Base(IndexedAttestationBase { attesting_indices: attesting_indices.as_ref().to_vec().into(), data: AttestationData { @@ -121,11 +120,9 @@ pub fn slashed_validators_from_attestations( continue; } - // TODO(electra) in a nested loop, copying to vecs feels bad - let attesting_indices_1 = att1.attesting_indices_to_vec(); - let attesting_indices_2 = att2.attesting_indices_to_vec(); - if att1.is_double_vote(att2) || att1.is_surround_vote(att2) { + let attesting_indices_1 = att1.attesting_indices_to_vec(); + let attesting_indices_2 = att2.attesting_indices_to_vec(); slashed_validators.extend(hashset_intersection( &attesting_indices_1, &attesting_indices_2, From 3ac3ddb2b7d4e4a824593db0a57fc6f9e39ec710 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 18 Jun 2024 00:23:02 +1000 Subject: [PATCH 130/227] Clean up Electra observed aggregates (#5929) * Use consistent key in observed_attestations * Remove unwraps from observed aggregates --- .../src/attestation_verification.rs | 15 ++-- beacon_node/beacon_chain/src/lib.rs | 2 +- .../beacon_chain/src/observed_aggregates.rs | 74 +++++++++++++------ .../tests/attestation_verification.rs | 6 +- 4 files changed, 61 insertions(+), 36 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index d82685c48c0..f2e3565ae66 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -35,8 +35,10 @@ mod batch; use crate::{ - beacon_chain::VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT, metrics, - observed_aggregates::ObserveOutcome, observed_attesters::Error as ObservedAttestersError, + beacon_chain::VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT, + metrics, + observed_aggregates::{ObserveOutcome, ObservedAttestationKey}, + observed_attesters::Error as ObservedAttestersError, BeaconChain, BeaconChainError, BeaconChainTypes, }; use bls::verify_signature_sets; @@ -58,9 +60,8 @@ use state_processing::{ use std::borrow::Cow; use strum::AsRefStr; use tree_hash::TreeHash; -use tree_hash_derive::TreeHash; use types::{ - Attestation, AttestationData, AttestationRef, BeaconCommittee, BeaconStateError, + Attestation, AttestationRef, BeaconCommittee, BeaconStateError, BeaconStateError::NoCommitteeFound, ChainSpec, CommitteeIndex, Epoch, EthSpec, ForkName, Hash256, IndexedAttestation, SelectionProof, SignedAggregateAndProof, Slot, SubnetId, }; @@ -309,12 +310,6 @@ struct IndexedAggregatedAttestation<'a, T: BeaconChainTypes> { observed_attestation_key_root: Hash256, } -#[derive(TreeHash)] -pub struct ObservedAttestationKey { - pub committee_index: u64, - pub attestation_data: AttestationData, -} - /// Wraps a `Attestation` that has been verified up until the point that an `IndexedAttestation` can /// be derived. /// diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index f419429e090..466ab0b67e7 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -39,7 +39,7 @@ mod light_client_server_cache; pub mod metrics; pub mod migrate; mod naive_aggregation_pool; -mod observed_aggregates; +pub mod observed_aggregates; mod observed_attesters; mod observed_blob_sidecars; pub mod observed_block_producers; diff --git a/beacon_node/beacon_chain/src/observed_aggregates.rs b/beacon_node/beacon_chain/src/observed_aggregates.rs index 5e161a998b5..b02bff96a2d 100644 --- a/beacon_node/beacon_chain/src/observed_aggregates.rs +++ b/beacon_node/beacon_chain/src/observed_aggregates.rs @@ -6,11 +6,14 @@ use ssz_types::{BitList, BitVector}; use std::collections::HashMap; use std::marker::PhantomData; use tree_hash::TreeHash; +use tree_hash_derive::TreeHash; use types::consts::altair::{ SYNC_COMMITTEE_SUBNET_COUNT, TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE, }; use types::slot_data::SlotData; -use types::{Attestation, AttestationRef, EthSpec, Hash256, Slot, SyncCommitteeContribution}; +use types::{ + Attestation, AttestationData, AttestationRef, EthSpec, Hash256, Slot, SyncCommitteeContribution, +}; pub type ObservedSyncContributions = ObservedAggregates< SyncCommitteeContribution, @@ -20,6 +23,17 @@ pub type ObservedSyncContributions = ObservedAggregates< pub type ObservedAggregateAttestations = ObservedAggregates, E, BitList<::MaxValidatorsPerSlot>>; +/// Attestation data augmented with committee index +/// +/// This is hashed and used to key the map of observed aggregate attestations. This is important +/// post-Electra where the attestation data committee index is 0 and we want to avoid accidentally +/// comparing aggregation bits for *different* committees. +#[derive(TreeHash)] +pub struct ObservedAttestationKey { + pub committee_index: u64, + pub attestation_data: AttestationData, +} + /// A trait use to associate capacity constants with the type being stored in `ObservedAggregates`. pub trait Consts { /// The default capacity of items stored per slot, in a single `SlotHashSet`. @@ -92,11 +106,11 @@ pub trait SubsetItem { /// Returns the item that gets stored in `ObservedAggregates` for later subset /// comparison with incoming aggregates. - fn get_item(&self) -> Self::Item; + fn get_item(&self) -> Result; /// Returns a unique value that keys the object to the item that is being stored /// in `ObservedAggregates`. - fn root(&self) -> Hash256; + fn root(&self) -> Result; } impl<'a, E: EthSpec> SubsetItem for AttestationRef<'a, E> { @@ -126,19 +140,22 @@ impl<'a, E: EthSpec> SubsetItem for AttestationRef<'a, E> { } /// Returns the sync contribution aggregation bits. - fn get_item(&self) -> Self::Item { + fn get_item(&self) -> Result { match self { - Self::Base(att) => { - // TODO(electra) fix unwrap - att.extend_aggregation_bits().unwrap() - } - Self::Electra(att) => att.aggregation_bits.clone(), + Self::Base(att) => att + .extend_aggregation_bits() + .map_err(|_| Error::GetItemError), + Self::Electra(att) => Ok(att.aggregation_bits.clone()), } } - /// Returns the hash tree root of the attestation data. - fn root(&self) -> Hash256 { - self.data().tree_hash_root() + /// Returns the hash tree root of the attestation data augmented with the committee index. + fn root(&self) -> Result { + Ok(ObservedAttestationKey { + committee_index: self.committee_index().ok_or(Error::RootError)?, + attestation_data: self.data().clone(), + } + .tree_hash_root()) } } @@ -153,19 +170,19 @@ impl<'a, E: EthSpec> SubsetItem for &'a SyncCommitteeContribution { } /// Returns the sync contribution aggregation bits. - fn get_item(&self) -> Self::Item { - self.aggregation_bits.clone() + fn get_item(&self) -> Result { + Ok(self.aggregation_bits.clone()) } /// Returns the hash tree root of the root, slot and subcommittee index /// of the sync contribution. - fn root(&self) -> Hash256 { - SyncCommitteeData { + fn root(&self) -> Result { + Ok(SyncCommitteeData { root: self.beacon_block_root, slot: self.slot, subcommittee_index: self.subcommittee_index, } - .tree_hash_root() + .tree_hash_root()) } } @@ -192,6 +209,8 @@ pub enum Error { expected: Slot, attestation: Slot, }, + GetItemError, + RootError, } /// A `HashMap` that contains entries related to some `Slot`. @@ -234,7 +253,7 @@ impl SlotHashSet { // If true, we replace the new item with its existing subset. This allows us // to hold fewer items in the list. } else if item.is_superset(existing) { - *existing = item.get_item(); + *existing = item.get_item()?; return Ok(ObserveOutcome::New); } } @@ -252,7 +271,7 @@ impl SlotHashSet { return Err(Error::ReachedMaxObservationsPerSlot(self.max_capacity)); } - let item = item.get_item(); + let item = item.get_item()?; self.map.entry(root).or_default().push(item); Ok(ObserveOutcome::New) } @@ -345,7 +364,7 @@ where root_opt: Option, ) -> Result { let index = self.get_set_index(item.get_slot())?; - let root = root_opt.unwrap_or_else(|| item.root()); + let root = root_opt.map_or_else(|| item.root(), Ok)?; self.sets .get_mut(index) @@ -487,7 +506,10 @@ mod tests { for a in &items { assert_eq!( - store.is_known_subset(a.as_reference(), a.as_reference().root()), + store.is_known_subset( + a.as_reference(), + a.as_reference().root().unwrap() + ), Ok(false), "should indicate an unknown attestation is unknown" ); @@ -500,12 +522,18 @@ mod tests { for a in &items { assert_eq!( - store.is_known_subset(a.as_reference(), a.as_reference().root()), + store.is_known_subset( + a.as_reference(), + a.as_reference().root().unwrap() + ), Ok(true), "should indicate a known attestation is known" ); assert_eq!( - store.observe_item(a.as_reference(), Some(a.as_reference().root())), + store.observe_item( + a.as_reference(), + Some(a.as_reference().root().unwrap()) + ), Ok(ObserveOutcome::Subset), "should acknowledge an existing attestation" ); diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index 63740c4736c..3e35e58296a 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -2,8 +2,8 @@ use beacon_chain::attestation_verification::{ batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations, Error, - ObservedAttestationKey, }; +use beacon_chain::observed_aggregates::ObservedAttestationKey; use beacon_chain::test_utils::{MakeAttestationOptions, HARNESS_GENESIS_TIME}; use beacon_chain::{ attestation_verification::Error as AttnError, @@ -852,7 +852,9 @@ async fn aggregated_gossip_verification() { err, AttnError::AttestationSupersetKnown(hash) if hash == ObservedAttestationKey { - committee_index: tester.valid_aggregate.message().aggregate().expect("should get committee index"), + committee_index: tester.valid_aggregate.message().aggregate() + .committee_index() + .expect("should get committee index"), attestation_data: tester.valid_aggregate.message().aggregate().data().clone(), }.tree_hash_root() )) From 1b908f59ec0f1c1eff0b7f201fd70b9b3e3de71c Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Mon, 17 Jun 2024 17:04:28 +0200 Subject: [PATCH 131/227] move committee bits to the last index in attestation (#5924) --- consensus/types/src/attestation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 3df14feadef..eccc63e6be3 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -69,9 +69,9 @@ pub struct Attestation { #[superstruct(only(Electra), partial_getter(rename = "aggregation_bits_electra"))] pub aggregation_bits: BitList, pub data: AttestationData, + pub signature: AggregateSignature, #[superstruct(only(Electra))] pub committee_bits: BitVector, - pub signature: AggregateSignature, } // TODO(electra): think about how to handle fork variants here From d87541c0457a9cf2b89a83a3750b49a651ed2489 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 15:44:32 +0200 Subject: [PATCH 132/227] De-dup attestation constructor logic --- consensus/types/src/aggregate_and_proof.rs | 46 ++++++++++++------- .../types/src/signed_aggregate_and_proof.rs | 14 +++--- validator_client/src/validator_store.rs | 44 +++++------------- 3 files changed, 48 insertions(+), 56 deletions(-) diff --git a/consensus/types/src/aggregate_and_proof.rs b/consensus/types/src/aggregate_and_proof.rs index e8fa01c143e..ae80369130a 100644 --- a/consensus/types/src/aggregate_and_proof.rs +++ b/consensus/types/src/aggregate_and_proof.rs @@ -4,6 +4,7 @@ use super::{ SignedRoot, }; use crate::test_utils::TestRandom; +use crate::Attestation; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; @@ -88,28 +89,39 @@ impl AggregateAndProof { genesis_validators_root: Hash256, spec: &ChainSpec, ) -> Self { - let selection_proof = selection_proof - .unwrap_or_else(|| { - SelectionProof::new::( - aggregate.data().slot, - secret_key, - fork, - genesis_validators_root, - spec, - ) - }) - .into(); + let selection_proof = selection_proof.unwrap_or_else(|| { + SelectionProof::new::( + aggregate.data().slot, + secret_key, + fork, + genesis_validators_root, + spec, + ) + }); + Self::from_attestation( + aggregator_index, + aggregate.clone_as_attestation(), + selection_proof, + ) + } + + /// Produces a new `AggregateAndProof` given a `selection_proof` + pub fn from_attestation( + aggregator_index: u64, + aggregate: Attestation, + selection_proof: SelectionProof, + ) -> Self { match aggregate { - AttestationRef::Base(attestation) => Self::Base(AggregateAndProofBase { + Attestation::Base(aggregate) => Self::Base(AggregateAndProofBase { aggregator_index, - aggregate: attestation.clone(), - selection_proof, + aggregate, + selection_proof: selection_proof.into(), }), - AttestationRef::Electra(attestation) => Self::Electra(AggregateAndProofElectra { + Attestation::Electra(aggregate) => Self::Electra(AggregateAndProofElectra { aggregator_index, - aggregate: attestation.clone(), - selection_proof, + aggregate, + selection_proof: selection_proof.into(), }), } } diff --git a/consensus/types/src/signed_aggregate_and_proof.rs b/consensus/types/src/signed_aggregate_and_proof.rs index ddf1dedb040..94624f02fe7 100644 --- a/consensus/types/src/signed_aggregate_and_proof.rs +++ b/consensus/types/src/signed_aggregate_and_proof.rs @@ -83,17 +83,19 @@ impl SignedAggregateAndProof { ); let signing_message = message.signing_root(domain); - match message { + Self::from_aggregate_and_proof(message, secret_key.sign(signing_message)) + } + + /// Produces a new `SignedAggregateAndProof` given a `signature` of `aggregate` + pub fn from_aggregate_and_proof(aggregate: AggregateAndProof, signature: Signature) -> Self { + match aggregate { AggregateAndProof::Base(message) => { - SignedAggregateAndProof::Base(SignedAggregateAndProofBase { - message, - signature: secret_key.sign(signing_message), - }) + SignedAggregateAndProof::Base(SignedAggregateAndProofBase { message, signature }) } AggregateAndProof::Electra(message) => { SignedAggregateAndProof::Electra(SignedAggregateAndProofElectra { message, - signature: secret_key.sign(signing_message), + signature, }) } } diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index a173e5da12b..e6e19b6e06c 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -18,16 +18,12 @@ use std::sync::Arc; use task_executor::TaskExecutor; use types::{ attestation::Error as AttestationError, graffiti::GraffitiString, AbstractExecPayload, Address, - Attestation, BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof, Domain, Epoch, - EthSpec, Fork, ForkName, Graffiti, Hash256, PublicKeyBytes, SelectionProof, Signature, - SignedBeaconBlock, SignedContributionAndProof, SignedRoot, SignedValidatorRegistrationData, - SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, - SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, - VoluntaryExit, -}; -use types::{ - AggregateAndProof, AggregateAndProofBase, AggregateAndProofElectra, SignedAggregateAndProof, - SignedAggregateAndProofBase, SignedAggregateAndProofElectra, + AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof, + Domain, Epoch, EthSpec, Fork, ForkName, Graffiti, Hash256, PublicKeyBytes, SelectionProof, + Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedRoot, + SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, + SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, + ValidatorRegistrationData, VoluntaryExit, }; pub use crate::doppelganger_service::DoppelgangerStatus; @@ -805,18 +801,8 @@ impl ValidatorStore { let signing_epoch = aggregate.data().target.epoch; let signing_context = self.signing_context(Domain::AggregateAndProof, signing_epoch); - let message = match aggregate { - Attestation::Base(att) => AggregateAndProof::Base(AggregateAndProofBase { - aggregator_index, - aggregate: att, - selection_proof: selection_proof.into(), - }), - Attestation::Electra(att) => AggregateAndProof::Electra(AggregateAndProofElectra { - aggregator_index, - aggregate: att, - selection_proof: selection_proof.into(), - }), - }; + let message = + AggregateAndProof::from_attestation(aggregator_index, aggregate, selection_proof); let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method @@ -830,17 +816,9 @@ impl ValidatorStore { metrics::inc_counter_vec(&metrics::SIGNED_AGGREGATES_TOTAL, &[metrics::SUCCESS]); - match message { - AggregateAndProof::Base(message) => { - Ok(SignedAggregateAndProof::Base(SignedAggregateAndProofBase { - message, - signature, - })) - } - AggregateAndProof::Electra(message) => Ok(SignedAggregateAndProof::Electra( - SignedAggregateAndProofElectra { message, signature }, - )), - } + Ok(SignedAggregateAndProof::from_aggregate_and_proof( + message, signature, + )) } /// Produces a `SelectionProof` for the `slot`, signed by with corresponding secret key to From dd0d5e2d9332c39d998b670bd26789e68d3e119a Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:17:04 +0200 Subject: [PATCH 133/227] Remove unwraps in Attestation construction --- consensus/types/src/attestation.rs | 31 ++++++++++++++++++ validator_client/src/attestation_service.rs | 36 ++++++++++----------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 3df14feadef..f4d674fd78c 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -1,4 +1,5 @@ use crate::slot_data::SlotData; +use crate::ForkName; use crate::{test_utils::TestRandom, Hash256, Slot}; use derivative::Derivative; use rand::RngCore; @@ -22,6 +23,8 @@ pub enum Error { AlreadySigned(usize), SubnetCountIsZero(ArithError), IncorrectStateVariant, + InvalidCommitteeLength, + InvalidCommitteeIndex, } #[superstruct( @@ -104,6 +107,34 @@ impl Hash for Attestation { } impl Attestation { + pub fn empty_for_signing( + committee_index: usize, + committee_length: usize, + data: AttestationData, + spec: &ChainSpec, + ) -> Result { + if spec.fork_name_at_slot::(data.slot) >= ForkName::Electra { + let mut committee_bits: BitVector = BitVector::default(); + committee_bits + .set(committee_index, true) + .map_err(|_| Error::InvalidCommitteeIndex)?; + Ok(Attestation::Electra(AttestationElectra { + aggregation_bits: BitList::with_capacity(committee_length) + .map_err(|_| Error::InvalidCommitteeLength)?, + data, + committee_bits, + signature: AggregateSignature::infinity(), + })) + } else { + Ok(Attestation::Base(AttestationBase { + aggregation_bits: BitList::with_capacity(committee_length) + .map_err(|_| Error::InvalidCommitteeLength)?, + data, + signature: AggregateSignature::infinity(), + })) + } + } + /// Aggregate another Attestation into this one. /// /// The aggregation bitfields must be disjoint, and the data must be the same. diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 404fe41249b..a3ab7a4c726 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -386,25 +386,23 @@ impl AttestationService { return None; } - let mut attestation = if fork_name >= ForkName::Electra { - let mut committee_bits: BitVector = BitVector::default(); - committee_bits - .set(duty.committee_index as usize, true) - .unwrap(); - Attestation::Electra(AttestationElectra { - aggregation_bits: BitList::with_capacity(duty.committee_length as usize) - .unwrap(), - data: attestation_data.clone(), - committee_bits, - signature: AggregateSignature::infinity(), - }) - } else { - Attestation::Base(AttestationBase { - aggregation_bits: BitList::with_capacity(duty.committee_length as usize) - .unwrap(), - data: attestation_data.clone(), - signature: AggregateSignature::infinity(), - }) + let mut attestation = match Attestation::::empty_for_signing( + duty.committee_index as usize, + duty.committee_length as usize, + attestation_data.clone(), + &self.context.eth2_config.spec, + ) { + Ok(attestation) => attestation, + Err(err) => { + crit!( + log, + "Invalid validator duties during signing"; + "validator" => ?duty.pubkey, + "duty" => ?duty, + "err" => ?err, + ); + return None; + } }; match self From 3ec21a2435ae60bfbea28010507d0a74554c6b13 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:26:47 +0200 Subject: [PATCH 134/227] Dedup match_attestation_data --- common/eth2/src/types.rs | 15 +++++++++++ validator_client/src/attestation_service.rs | 30 +++------------------ 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 2bb749af9f6..c20e723dddb 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -716,6 +716,21 @@ pub struct AttesterData { pub slot: Slot, } +impl AttesterData { + pub fn match_attestation_data( + &self, + attestation_data: &AttestationData, + spec: &ChainSpec, + ) -> bool { + if spec.fork_name_at_slot::(attestation_data.slot) < ForkName::Electra { + self.slot == attestation_data.slot && self.committee_index == attestation_data.index + } else { + // After electra `attestation_data.index` is set to 0 and does not match the duties + self.slot == attestation_data.index + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ProposerData { pub pubkey: PublicKeyBytes, diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index a3ab7a4c726..d08d7567afe 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -14,11 +14,7 @@ use std::ops::Deref; use std::sync::Arc; use tokio::time::{sleep, sleep_until, Duration, Instant}; use tree_hash::TreeHash; -use types::ForkName; -use types::{ - attestation::AttestationBase, AggregateSignature, Attestation, AttestationData, - AttestationElectra, BitList, BitVector, ChainSpec, CommitteeIndex, EthSpec, Slot, -}; +use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot}; /// Builds an `AttestationService`. pub struct AttestationServiceBuilder { @@ -363,17 +359,8 @@ impl AttestationService { let duty = &duty_and_proof.duty; let attestation_data = attestation_data_ref; - let fork_name = self - .context - .eth2_config - .spec - .fork_name_at_slot::(attestation_data.slot); - // Ensure that the attestation matches the duties. - #[allow(clippy::suspicious_operation_groupings)] - if duty.slot != attestation_data.slot - || (fork_name < ForkName::Electra && duty.committee_index != attestation_data.index) - { + if !duty.match_attestation_data::(attestation_data, &self.context.eth2_config.spec) { crit!( log, "Inconsistent validator duties during signing"; @@ -552,23 +539,12 @@ impl AttestationService { .await .map_err(|e| e.to_string())?; - let fork_name = self - .context - .eth2_config - .spec - .fork_name_at_slot::(attestation_data.slot); - // Create futures to produce the signed aggregated attestations. let signing_futures = validator_duties.iter().map(|duty_and_proof| async move { let duty = &duty_and_proof.duty; let selection_proof = duty_and_proof.selection_proof.as_ref()?; - let slot = attestation_data.slot; - let committee_index = attestation_data.index; - - if duty.slot != slot - || (fork_name < ForkName::Electra && duty.committee_index != committee_index) - { + if !duty.match_attestation_data::(attestation_data, &self.context.eth2_config.spec) { crit!(log, "Inconsistent validator duties during signing"); return None; } From 795eff9bf49fd58d2a707c10a07a9e1622c5df0d Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:58:26 +0200 Subject: [PATCH 135/227] Remove outdated TODO --- testing/ef_tests/src/cases/fork_choice.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index b3b20d6efe1..fa8c13d07ec 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -131,7 +131,6 @@ pub struct ForkChoiceTest { pub anchor_state: BeaconState, pub anchor_block: BeaconBlock, #[allow(clippy::type_complexity)] - // TODO(electra): these tests will need to be updated to use new types pub steps: Vec< Step, BlobsList, Attestation, AttesterSlashing, PowBlock>, >, From 960f8c5c48c708df1a123e045cf4fc17398fb2d3 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:58:38 +0200 Subject: [PATCH 136/227] Use ForkName Ord in fork-choice tests --- testing/ef_tests/src/cases/fork_choice.rs | 50 ++++++++++------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index fa8c13d07ec..d6c8b6302d1 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -179,42 +179,34 @@ impl LoadCase for ForkChoiceTest { valid, }) } - Step::Attestation { attestation } => match fork_name { - ForkName::Base - | ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb => ssz_decode_file( - &path.join(format!("{}.ssz_snappy", attestation)), - ) - .map(|attestation| Step::Attestation { - attestation: Attestation::Base(attestation), - }), - ForkName::Electra => ssz_decode_file( - &path.join(format!("{}.ssz_snappy", attestation)), - ) - .map(|attestation| Step::Attestation { - attestation: Attestation::Electra(attestation), - }), - }, - Step::AttesterSlashing { attester_slashing } => match fork_name { - ForkName::Base - | ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb => { + Step::Attestation { attestation } => { + if fork_name >= ForkName::Electra { + ssz_decode_file(&path.join(format!("{}.ssz_snappy", attestation))).map( + |attestation| Step::Attestation { + attestation: Attestation::Electra(attestation), + }, + ) + } else { + ssz_decode_file(&path.join(format!("{}.ssz_snappy", attestation))).map( + |attestation| Step::Attestation { + attestation: Attestation::Base(attestation), + }, + ) + } + } + Step::AttesterSlashing { attester_slashing } => { + if fork_name >= ForkName::Electra { ssz_decode_file(&path.join(format!("{}.ssz_snappy", attester_slashing))) .map(|attester_slashing| Step::AttesterSlashing { - attester_slashing: AttesterSlashing::Base(attester_slashing), + attester_slashing: AttesterSlashing::Electra(attester_slashing), }) - } - ForkName::Electra => { + } else { ssz_decode_file(&path.join(format!("{}.ssz_snappy", attester_slashing))) .map(|attester_slashing| Step::AttesterSlashing { - attester_slashing: AttesterSlashing::Electra(attester_slashing), + attester_slashing: AttesterSlashing::Base(attester_slashing), }) } - }, + } Step::PowBlock { pow_block } => { ssz_decode_file(&path.join(format!("{}.ssz_snappy", pow_block))) .map(|pow_block| Step::PowBlock { pow_block }) From 1d0e3f4d30ae31253f3f9b45822b18731ac9032a Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:18:54 +0200 Subject: [PATCH 137/227] Use ForkName Ord in BeaconBlockBody --- consensus/types/src/beacon_block_body.rs | 27 ++++++++++++------------ 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index 76a2fc1872e..9cf66184f8d 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -820,6 +820,11 @@ impl From>> } impl BeaconBlockBody { + /// Returns the name of the fork pertaining to `self`. + pub fn fork_name(&self) -> ForkName { + self.to_ref().fork_name() + } + pub fn block_body_merkle_proof(&self, generalized_index: usize) -> Result, Error> { let field_index = match generalized_index { light_client_update::EXECUTION_PAYLOAD_INDEX => { @@ -834,22 +839,16 @@ impl BeaconBlockBody { _ => return Err(Error::IndexNotSupported(generalized_index)), }; - let attestations_root = match self { - BeaconBlockBody::Base(_) - | BeaconBlockBody::Altair(_) - | BeaconBlockBody::Bellatrix(_) - | BeaconBlockBody::Capella(_) - | BeaconBlockBody::Deneb(_) => self.attestations_base()?.tree_hash_root(), - BeaconBlockBody::Electra(_) => self.attestations_electra()?.tree_hash_root(), + let attestations_root = if self.fork_name() > ForkName::Electra { + self.attestations_electra()?.tree_hash_root() + } else { + self.attestations_base()?.tree_hash_root() }; - let attester_slashings_root = match self { - BeaconBlockBody::Base(_) - | BeaconBlockBody::Altair(_) - | BeaconBlockBody::Bellatrix(_) - | BeaconBlockBody::Capella(_) - | BeaconBlockBody::Deneb(_) => self.attester_slashings_base()?.tree_hash_root(), - BeaconBlockBody::Electra(_) => self.attester_slashings_electra()?.tree_hash_root(), + let attester_slashings_root = if self.fork_name() > ForkName::Electra { + self.attester_slashings_electra()?.tree_hash_root() + } else { + self.attester_slashings_base()?.tree_hash_root() }; let mut leaves = vec![ From 5acc0523dfb61ece807814ffe2260a8b5eabf760 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 18:18:44 +0200 Subject: [PATCH 138/227] Make to_electra not fallible --- consensus/types/src/indexed_attestation.rs | 10 ++++---- slasher/src/array.rs | 20 ++-------------- slasher/src/lib.rs | 26 ++++++-------------- slasher/src/slasher.rs | 28 +++++++--------------- slasher/src/test_utils.rs | 4 ++-- 5 files changed, 25 insertions(+), 63 deletions(-) diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index d282e2f259c..ecddcd21796 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -116,11 +116,13 @@ impl IndexedAttestation { } } - pub fn to_electra(self) -> Result, ssz_types::Error> { - Ok(match self { + pub fn to_electra(self) -> IndexedAttestationElectra { + match self { Self::Base(att) => { let extended_attesting_indices: VariableList = - VariableList::new(att.attesting_indices.to_vec())?; + VariableList::new(att.attesting_indices.to_vec()) + .expect("MaxValidatorsPerSlot must be >= MaxValidatorsPerCommittee"); + // TODO: Add test after unstable rebase https://github.com/sigp/lighthouse/blob/474c1b44863927c588dd05ab2ac0f934298398e1/consensus/types/src/eth_spec.rs#L541 IndexedAttestationElectra { attesting_indices: extended_attesting_indices, @@ -129,7 +131,7 @@ impl IndexedAttestation { } } Self::Electra(att) => att, - }) + } } } diff --git a/slasher/src/array.rs b/slasher/src/array.rs index 714ccf4e77b..77ddceb85fe 100644 --- a/slasher/src/array.rs +++ b/slasher/src/array.rs @@ -5,7 +5,6 @@ use crate::{ }; use flate2::bufread::{ZlibDecoder, ZlibEncoder}; use serde::{Deserialize, Serialize}; -use slog::Logger; use std::borrow::Borrow; use std::collections::{btree_map::Entry, BTreeMap, HashSet}; use std::io::Read; @@ -486,7 +485,6 @@ pub fn update( batch: Vec>>, current_epoch: Epoch, config: &Config, - log: &Logger, ) -> Result>, Error> { // Split the batch up into horizontal segments. // Map chunk indexes in the range `0..self.config.chunk_size` to attestations @@ -506,7 +504,6 @@ pub fn update( &chunk_attestations, current_epoch, config, - log, )?; slashings.extend(update_array::<_, MaxTargetChunk>( db, @@ -515,7 +512,6 @@ pub fn update( &chunk_attestations, current_epoch, config, - log, )?); // Update all current epochs. @@ -574,7 +570,6 @@ pub fn update_array( chunk_attestations: &BTreeMap>>>, current_epoch: Epoch, config: &Config, - log: &Logger, ) -> Result>, Error> { let mut slashings = HashSet::new(); // Map from chunk index to updated chunk at that index. @@ -608,19 +603,8 @@ pub fn update_array( current_epoch, config, )?; - match slashing_status.into_slashing(&attestation.indexed) { - Ok(Some(slashing)) => { - slashings.insert(slashing); - } - Err(e) => { - slog::error!( - log, - "Invalid slashing conversion"; - "validator_index" => validator_index, - "error" => e - ); - } - Ok(None) => {} + if let Some(slashing) = slashing_status.into_slashing(&attestation.indexed) { + slashings.insert(slashing); } } } diff --git a/slasher/src/lib.rs b/slasher/src/lib.rs index 0c097ac77fb..4d58fa77029 100644 --- a/slasher/src/lib.rs +++ b/slasher/src/lib.rs @@ -53,11 +53,11 @@ impl AttesterSlashingStatus { pub fn into_slashing( self, new_attestation: &IndexedAttestation, - ) -> Result>, String> { + ) -> Option> { use AttesterSlashingStatus::*; // The surrounding attestation must be in `attestation_1` to be valid. - Ok(match self { + match self { NotSlashable => None, AlreadyDoubleVoted => None, DoubleVote(existing) | SurroundedByExisting(existing) => { @@ -70,14 +70,8 @@ impl AttesterSlashingStatus { } // A slashing involving an electra attestation type must return an `AttesterSlashingElectra` type (_, _) => Some(AttesterSlashing::Electra(AttesterSlashingElectra { - attestation_1: existing - .clone() - .to_electra() - .map_err(|e| format!("{e:?}"))?, - attestation_2: new_attestation - .clone() - .to_electra() - .map_err(|e| format!("{e:?}"))?, + attestation_1: existing.clone().to_electra(), + attestation_2: new_attestation.clone().to_electra(), })), } } @@ -90,16 +84,10 @@ impl AttesterSlashingStatus { } // A slashing involving an electra attestation type must return an `AttesterSlashingElectra` type (_, _) => Some(AttesterSlashing::Electra(AttesterSlashingElectra { - attestation_1: new_attestation - .clone() - .to_electra() - .map_err(|e| format!("{e:?}"))?, - attestation_2: existing - .clone() - .to_electra() - .map_err(|e| format!("{e:?}"))?, + attestation_1: new_attestation.clone().to_electra(), + attestation_2: existing.clone().to_electra(), })), }, - }) + } } } diff --git a/slasher/src/slasher.rs b/slasher/src/slasher.rs index ef6dcce9b1b..fc8e8453c89 100644 --- a/slasher/src/slasher.rs +++ b/slasher/src/slasher.rs @@ -247,7 +247,6 @@ impl Slasher { batch, current_epoch, &self.config, - &self.log, ) { Ok(slashings) => { if !slashings.is_empty() { @@ -295,25 +294,14 @@ impl Slasher { indexed_attestation_id, )?; - match slashing_status.into_slashing(attestation) { - Ok(Some(slashing)) => { - debug!( - self.log, - "Found double-vote slashing"; - "validator_index" => validator_index, - "epoch" => slashing.attestation_1().data().target.epoch, - ); - slashings.insert(slashing); - } - Err(e) => { - error!( - self.log, - "Invalid slashing conversion"; - "validator_index" => validator_index, - "error" => e - ); - } - Ok(None) => {} + if let Some(slashing) = slashing_status.into_slashing(attestation) { + debug!( + self.log, + "Found double-vote slashing"; + "validator_index" => validator_index, + "epoch" => slashing.attestation_1().data().target.epoch, + ); + slashings.insert(slashing); } } diff --git a/slasher/src/test_utils.rs b/slasher/src/test_utils.rs index ef623911bfa..7019a8aed76 100644 --- a/slasher/src/test_utils.rs +++ b/slasher/src/test_utils.rs @@ -71,8 +71,8 @@ pub fn att_slashing( } // A slashing involving an electra attestation type must return an electra AttesterSlashing type (_, _) => AttesterSlashing::Electra(AttesterSlashingElectra { - attestation_1: attestation_1.clone().to_electra().unwrap(), - attestation_2: attestation_2.clone().to_electra().unwrap(), + attestation_1: attestation_1.clone().to_electra(), + attestation_2: attestation_2.clone().to_electra(), }), } } From 4f08f6e0da46aefc00a5be47465323395f45b08f Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 18:34:54 +0200 Subject: [PATCH 139/227] Remove TestRandom impl for IndexedAttestation --- consensus/types/src/indexed_attestation.rs | 30 ++++++++-------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index ecddcd21796..1af2512da8b 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -1,7 +1,6 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, EthSpec, VariableList}; use core::slice::Iter; use derivative::Derivative; -use rand::RngCore; use serde::{Deserialize, Serialize}; use ssz::Decode; use ssz::Encode; @@ -208,23 +207,6 @@ impl Decode for IndexedAttestation { } } -// TODO(electra): think about how to handle fork variants here -impl TestRandom for IndexedAttestation { - fn random_for_test(rng: &mut impl RngCore) -> Self { - let attesting_indices = VariableList::random_for_test(rng); - // let committee_bits: BitList = BitList::random_for_test(rng); - let data = AttestationData::random_for_test(rng); - let signature = AggregateSignature::random_for_test(rng); - - Self::Base(IndexedAttestationBase { - attesting_indices, - // committee_bits, - data, - signature, - }) - } -} - /// Implementation of non-crypto-secure `Hash`, for use with `HashMap` and `HashSet`. /// /// Guarantees `att1 == att2 -> hash(att1) == hash(att2)`. @@ -333,14 +315,22 @@ mod tests { assert!(!indexed_vote_first.is_surround_vote(&indexed_vote_second)); } - ssz_and_tree_hash_tests!(IndexedAttestation); + mod base { + use super::*; + ssz_and_tree_hash_tests!(IndexedAttestationBase); + } + mod electra { + use super::*; + ssz_and_tree_hash_tests!(IndexedAttestationElectra); + } fn create_indexed_attestation( target_epoch: u64, source_epoch: u64, ) -> IndexedAttestation { let mut rng = XorShiftRng::from_seed([42; 16]); - let mut indexed_vote = IndexedAttestation::random_for_test(&mut rng); + let mut indexed_vote = + IndexedAttestation::Base(IndexedAttestationBase::random_for_test(&mut rng)); indexed_vote.data_mut().source.epoch = Epoch::new(source_epoch); indexed_vote.data_mut().target.epoch = Epoch::new(target_epoch); From f0492852f3a4cb3ce468275760b7c25935ac1aa9 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:01:06 +0200 Subject: [PATCH 140/227] Remove IndexedAttestation faulty Decode impl --- consensus/types/src/indexed_attestation.rs | 21 --------------------- slasher/src/database.rs | 9 +++++++-- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index 1af2512da8b..00cac09abf3 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -2,7 +2,6 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, EthSpec use core::slice::Iter; use derivative::Derivative; use serde::{Deserialize, Serialize}; -use ssz::Decode; use ssz::Encode; use ssz_derive::{Decode, Encode}; use std::hash::{Hash, Hasher}; @@ -187,26 +186,6 @@ impl<'a, E: EthSpec> IndexedAttestationRef<'a, E> { } } -impl Decode for IndexedAttestation { - fn is_ssz_fixed_len() -> bool { - false - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - if let Ok(result) = IndexedAttestationBase::from_ssz_bytes(bytes) { - return Ok(IndexedAttestation::Base(result)); - } - - if let Ok(result) = IndexedAttestationElectra::from_ssz_bytes(bytes) { - return Ok(IndexedAttestation::Electra(result)); - } - - Err(ssz::DecodeError::BytesInvalid(String::from( - "bytes not valid for any fork variant", - ))) - } -} - /// Implementation of non-crypto-secure `Hash`, for use with `HashMap` and `HashSet`. /// /// Guarantees `att1 == att2 -> hash(att1) == hash(att2)`. diff --git a/slasher/src/database.rs b/slasher/src/database.rs index d80112c553d..16a67cd684f 100644 --- a/slasher/src/database.rs +++ b/slasher/src/database.rs @@ -18,7 +18,8 @@ use std::marker::PhantomData; use std::sync::Arc; use tree_hash::TreeHash; use types::{ - Epoch, EthSpec, Hash256, IndexedAttestation, ProposerSlashing, SignedBeaconBlockHeader, Slot, + Epoch, EthSpec, Hash256, IndexedAttestation, IndexedAttestationBase, ProposerSlashing, + SignedBeaconBlockHeader, Slot, }; /// Current database schema version, to check compatibility of on-disk DB with software. @@ -481,7 +482,11 @@ impl SlasherDB { .ok_or(Error::MissingIndexedAttestation { id: indexed_attestation_id.as_u64(), })?; - ssz_decode(bytes) + // TODO(electra): make slasher fork-aware and return correct variant of attestation. Both + // Base and Electra variant can the same SSZ encoding, however the smaller list maximum of + // Base can error with an Electra attestation with heavy participation. + let indexed_attestation: IndexedAttestationBase = ssz_decode(bytes)?; + Ok(IndexedAttestation::Base(indexed_attestation)) } fn get_attestation_data_root( From 5070ab254dd70b6de19a43669691529df6a68915 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:56:12 +0200 Subject: [PATCH 141/227] Drop TestRandom impl --- consensus/types/src/attestation.rs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index f4d674fd78c..21e0a7ccd8e 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -2,7 +2,6 @@ use crate::slot_data::SlotData; use crate::ForkName; use crate::{test_utils::TestRandom, Hash256, Slot}; use derivative::Derivative; -use rand::RngCore; use safe_arith::ArithError; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -77,23 +76,6 @@ pub struct Attestation { pub signature: AggregateSignature, } -// TODO(electra): think about how to handle fork variants here -impl TestRandom for Attestation { - fn random_for_test(rng: &mut impl RngCore) -> Self { - let aggregation_bits = BitList::random_for_test(rng); - let data = AttestationData::random_for_test(rng); - let signature = AggregateSignature::random_for_test(rng); - let committee_bits = BitVector::random_for_test(rng); - - Self::Electra(AttestationElectra { - aggregation_bits, - committee_bits, - data, - signature, - }) - } -} - impl Hash for Attestation { fn hash(&self, state: &mut H) where From 45d007a71f80cac6fd393d1de9b3cc2bd4e1fe04 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:01:19 +0200 Subject: [PATCH 142/227] Add PendingAttestationInElectra --- .../src/per_block_processing/errors.rs | 1 + .../process_operations.rs | 49 +++++++++---------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/errors.rs b/consensus/state_processing/src/per_block_processing/errors.rs index 336895514f9..71a284dea49 100644 --- a/consensus/state_processing/src/per_block_processing/errors.rs +++ b/consensus/state_processing/src/per_block_processing/errors.rs @@ -89,6 +89,7 @@ pub enum BlockProcessingError { found: Hash256, }, WithdrawalCredentialsInvalid, + PendingAttestationInElectra, } impl From for BlockProcessingError { diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 9b4e9d71394..bd354901a8b 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -76,33 +76,30 @@ pub mod base { ) .map_err(|e| e.into_with_index(i))?; - match attestation { - AttestationRef::Base(att) => { - let pending_attestation = PendingAttestation { - aggregation_bits: att.aggregation_bits.clone(), - data: att.data.clone(), - inclusion_delay: state.slot().safe_sub(att.data.slot)?.as_u64(), - proposer_index, - }; - - if attestation.data().target.epoch == state.current_epoch() { - state - .as_base_mut()? - .current_epoch_attestations - .push(pending_attestation)?; - } else { - state - .as_base_mut()? - .previous_epoch_attestations - .push(pending_attestation)?; - } - } - AttestationRef::Electra(_) => { - // TODO(electra) pending attestations are only phase 0 - // so we should just raise a relevant error here - todo!() - } + let AttestationRef::Base(attestation) = attestation else { + // Pending attestations have been deprecated in a altair, this branch should + // never happen + return Err(BlockProcessingError::PendingAttestationInElectra); }; + + let pending_attestation = PendingAttestation { + aggregation_bits: attestation.aggregation_bits.clone(), + data: attestation.data.clone(), + inclusion_delay: state.slot().safe_sub(attestation.data.slot)?.as_u64(), + proposer_index, + }; + + if attestation.data.target.epoch == state.current_epoch() { + state + .as_base_mut()? + .current_epoch_attestations + .push(pending_attestation)?; + } else { + state + .as_base_mut()? + .previous_epoch_attestations + .push(pending_attestation)?; + } } Ok(()) From 9e84779522633f544b314aadbcffe3af5f38c399 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 18 Jun 2024 11:10:32 -0400 Subject: [PATCH 143/227] Indexed att on disk (#35) * indexed att on disk * fix lints * Update slasher/src/migrate.rs Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com> --------- Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com> --- consensus/types/src/indexed_attestation.rs | 32 ++++++++++++++++++++++ consensus/types/src/lib.rs | 3 +- slasher/src/database.rs | 17 ++++++------ slasher/src/migrate.rs | 4 +++ 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index 00cac09abf3..429d65898a9 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -239,6 +239,38 @@ mod quoted_variable_list_u64 { } } +#[derive(Debug, Clone, Encode, Decode, PartialEq)] +#[ssz(enum_behaviour = "union")] +pub enum IndexedAttestationOnDisk { + Base(IndexedAttestationBase), + Electra(IndexedAttestationElectra), +} + +#[derive(Debug, Clone, Encode, PartialEq)] +#[ssz(enum_behaviour = "union")] +pub enum IndexedAttestationRefOnDisk<'a, E: EthSpec> { + Base(&'a IndexedAttestationBase), + Electra(&'a IndexedAttestationElectra), +} + +impl<'a, E: EthSpec> From<&'a IndexedAttestation> for IndexedAttestationRefOnDisk<'a, E> { + fn from(attestation: &'a IndexedAttestation) -> Self { + match attestation { + IndexedAttestation::Base(attestation) => Self::Base(attestation), + IndexedAttestation::Electra(attestation) => Self::Electra(attestation), + } + } +} + +impl From> for IndexedAttestation { + fn from(attestation: IndexedAttestationOnDisk) -> Self { + match attestation { + IndexedAttestationOnDisk::Base(attestation) => Self::Base(attestation), + IndexedAttestationOnDisk::Electra(attestation) => Self::Electra(attestation), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 75456446bf2..2a73ff0f37a 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -176,7 +176,8 @@ pub use crate::fork_versioned_response::{ForkVersionDeserialize, ForkVersionedRe pub use crate::graffiti::{Graffiti, GRAFFITI_BYTES_LEN}; pub use crate::historical_batch::HistoricalBatch; pub use crate::indexed_attestation::{ - IndexedAttestation, IndexedAttestationBase, IndexedAttestationElectra, IndexedAttestationRef, + IndexedAttestation, IndexedAttestationBase, IndexedAttestationElectra, + IndexedAttestationOnDisk, IndexedAttestationRef, IndexedAttestationRefOnDisk, }; pub use crate::light_client_bootstrap::{ LightClientBootstrap, LightClientBootstrapAltair, LightClientBootstrapCapella, diff --git a/slasher/src/database.rs b/slasher/src/database.rs index 16a67cd684f..16089706a00 100644 --- a/slasher/src/database.rs +++ b/slasher/src/database.rs @@ -18,12 +18,12 @@ use std::marker::PhantomData; use std::sync::Arc; use tree_hash::TreeHash; use types::{ - Epoch, EthSpec, Hash256, IndexedAttestation, IndexedAttestationBase, ProposerSlashing, - SignedBeaconBlockHeader, Slot, + Epoch, EthSpec, Hash256, IndexedAttestation, IndexedAttestationOnDisk, + IndexedAttestationRefOnDisk, ProposerSlashing, SignedBeaconBlockHeader, Slot, }; /// Current database schema version, to check compatibility of on-disk DB with software. -pub const CURRENT_SCHEMA_VERSION: u64 = 3; +pub const CURRENT_SCHEMA_VERSION: u64 = 4; /// Metadata about the slashing database itself. const METADATA_DB: &str = "metadata"; @@ -458,7 +458,9 @@ impl SlasherDB { }; let attestation_key = IndexedAttestationId::new(indexed_att_id); - let data = indexed_attestation.as_ssz_bytes(); + let indexed_attestation_on_disk: IndexedAttestationRefOnDisk = + indexed_attestation.into(); + let data = indexed_attestation_on_disk.as_ssz_bytes(); cursor.put(attestation_key.as_ref(), &data)?; drop(cursor); @@ -482,11 +484,8 @@ impl SlasherDB { .ok_or(Error::MissingIndexedAttestation { id: indexed_attestation_id.as_u64(), })?; - // TODO(electra): make slasher fork-aware and return correct variant of attestation. Both - // Base and Electra variant can the same SSZ encoding, however the smaller list maximum of - // Base can error with an Electra attestation with heavy participation. - let indexed_attestation: IndexedAttestationBase = ssz_decode(bytes)?; - Ok(IndexedAttestation::Base(indexed_attestation)) + let indexed_attestation: IndexedAttestationOnDisk = ssz_decode(bytes)?; + Ok(indexed_attestation.into()) } fn get_attestation_data_root( diff --git a/slasher/src/migrate.rs b/slasher/src/migrate.rs index 674ab9c132d..ce298caaf2c 100644 --- a/slasher/src/migrate.rs +++ b/slasher/src/migrate.rs @@ -17,6 +17,10 @@ impl SlasherDB { software_schema_version: CURRENT_SCHEMA_VERSION, }), (x, y) if x == y => Ok(self), + (3, 4) => { + // TODO(electra): db migration due to `IndexedAttestationOnDisk` + Ok(self) + } (_, _) => Err(Error::IncompatibleSchemaVersion { database_schema_version: schema_version, software_schema_version: CURRENT_SCHEMA_VERSION, From 7af3f2eb35d09aa12424f6f0bb641322eeba01b5 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Tue, 18 Jun 2024 22:50:07 +0200 Subject: [PATCH 144/227] add electra fork enabled fn to ForkName impl (#36) * add electra fork enabled fn to ForkName impl * remove inadvertent file --- .../beacon_chain/src/attestation_verification.rs | 5 ++++- beacon_node/beacon_chain/src/beacon_chain.rs | 6 +++++- beacon_node/beacon_chain/src/early_attester_cache.rs | 2 +- beacon_node/beacon_chain/src/test_utils.rs | 12 ++++++------ beacon_node/beacon_chain/tests/block_verification.rs | 2 +- beacon_node/beacon_chain/tests/store_tests.rs | 2 +- beacon_node/client/src/notifier.rs | 2 +- beacon_node/operation_pool/src/lib.rs | 2 +- consensus/types/src/attestation.rs | 3 +-- consensus/types/src/chain_spec.rs | 2 +- consensus/types/src/fork_name.rs | 4 ++++ consensus/types/src/validator.rs | 6 +++--- testing/ef_tests/src/cases/fork_choice.rs | 4 ++-- 13 files changed, 31 insertions(+), 21 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index f2e3565ae66..6eb0c6ecf9d 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -1341,7 +1341,10 @@ pub fn verify_committee_index( attestation: AttestationRef, spec: &ChainSpec, ) -> Result<(), Error> { - if spec.fork_name_at_slot::(attestation.data().slot) >= ForkName::Electra { + if spec + .fork_name_at_slot::(attestation.data().slot) + .electra_enabled() + { // Check to ensure that the attestation is for a single committee. let num_committee_bits = get_committee_indices::( attestation diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index abe14796ce7..eeb14579548 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1994,7 +1994,11 @@ impl BeaconChain { }; drop(cache_timer); - if self.spec.fork_name_at_slot::(request_slot) >= ForkName::Electra { + if self + .spec + .fork_name_at_slot::(request_slot) + .electra_enabled() + { let mut committee_bits = BitVector::default(); if committee_len > 0 { committee_bits.set(request_index as usize, true)?; diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index 936f4de3ee1..41a225d9f24 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -123,7 +123,7 @@ impl EarlyAttesterCache { item.committee_lengths .get_committee_length::(request_slot, request_index, spec)?; - let attestation = if spec.fork_name_at_slot::(request_slot) >= ForkName::Electra { + let attestation = if spec.fork_name_at_slot::(request_slot).electra_enabled() { let mut committee_bits = BitVector::default(); if committee_len > 0 { committee_bits diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 34ad6d43244..7d443db3809 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1033,7 +1033,7 @@ where *state.get_block_root(target_slot)? }; - if self.spec.fork_name_at_slot::(slot) >= ForkName::Electra { + if self.spec.fork_name_at_slot::(slot).electra_enabled() { let mut committee_bits = BitVector::default(); committee_bits.set(index as usize, true)?; Ok(Attestation::Electra(AttestationElectra { @@ -1376,7 +1376,7 @@ where let fork_name = self.spec.fork_name_at_slot::(slot); - let aggregate = if fork_name >= ForkName::Electra { + let aggregate = if fork_name.electra_enabled() { self.chain .get_aggregated_attestation_electra( slot, @@ -1541,7 +1541,7 @@ where let fork_name = self.spec.fork_name_at_slot::(Slot::new(0)); - let mut attestation_1 = if fork_name >= ForkName::Electra { + let mut attestation_1 = if fork_name.electra_enabled() { IndexedAttestation::Electra(IndexedAttestationElectra { attesting_indices: VariableList::new(validator_indices).unwrap(), data: AttestationData { @@ -1623,7 +1623,7 @@ where } } - if fork_name >= ForkName::Electra { + if fork_name.electra_enabled() { AttesterSlashing::Electra(AttesterSlashingElectra { attestation_1: attestation_1.as_electra().unwrap().clone(), attestation_2: attestation_2.as_electra().unwrap().clone(), @@ -1657,7 +1657,7 @@ where }, }; - let (mut attestation_1, mut attestation_2) = if fork_name >= ForkName::Electra { + let (mut attestation_1, mut attestation_2) = if fork_name.electra_enabled() { let attestation_1 = IndexedAttestationElectra { attesting_indices: VariableList::new(validator_indices_1).unwrap(), data: data.clone(), @@ -1735,7 +1735,7 @@ where } } - if fork_name >= ForkName::Electra { + if fork_name.electra_enabled() { AttesterSlashing::Electra(AttesterSlashingElectra { attestation_1: attestation_1.as_electra().unwrap().clone(), attestation_2: attestation_2.as_electra().unwrap().clone(), diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 0fe406a0172..55b2c14a3c5 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -668,7 +668,7 @@ async fn invalid_signature_attester_slashing() { let mut snapshots = chain_segment.clone(); let fork_name = harness.chain.spec.fork_name_at_slot::(Slot::new(0)); - let attester_slashing = if fork_name >= ForkName::Electra { + let attester_slashing = if fork_name.electra_enabled() { let indexed_attestation = IndexedAttestationElectra { attesting_indices: vec![0].into(), data: AttestationData { diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index ef873909309..c2468102eed 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -1025,7 +1025,7 @@ async fn multiple_attestations_per_block() { let slot = snapshot.beacon_block.slot(); let fork_name = harness.chain.spec.fork_name_at_slot::(slot); - if fork_name >= ForkName::Electra { + if fork_name.electra_enabled() { assert_eq!( snapshot .beacon_block diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index 7c224671ff3..632188014eb 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -551,7 +551,7 @@ async fn electra_readiness_logging( .snapshot .beacon_state .fork_name_unchecked() - >= ForkName::Electra; + .electra_enabled(); let has_execution_layer = beacon_chain.execution_layer.is_some(); diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index f9021bb258d..49fbed56869 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -286,7 +286,7 @@ impl OperationPool { // TODO(electra): Work out how to do this more elegantly. This is a bit of a hack. let mut all_attestations = self.attestations.write(); - if fork_name >= ForkName::Electra { + if fork_name.electra_enabled() { all_attestations.aggregate_across_committees(prev_epoch_key); all_attestations.aggregate_across_committees(curr_epoch_key); } diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 21e0a7ccd8e..35bd80fc334 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -1,5 +1,4 @@ use crate::slot_data::SlotData; -use crate::ForkName; use crate::{test_utils::TestRandom, Hash256, Slot}; use derivative::Derivative; use safe_arith::ArithError; @@ -95,7 +94,7 @@ impl Attestation { data: AttestationData, spec: &ChainSpec, ) -> Result { - if spec.fork_name_at_slot::(data.slot) >= ForkName::Electra { + if spec.fork_name_at_slot::(data.slot).electra_enabled() { let mut committee_bits: BitVector = BitVector::default(); committee_bits .set(committee_index, true) diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index b0346a14ef8..7609e360355 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -376,7 +376,7 @@ impl ChainSpec { state: &BeaconState, ) -> u64 { let fork_name = state.fork_name_unchecked(); - if fork_name >= ForkName::Electra { + if fork_name.electra_enabled() { self.min_slashing_penalty_quotient_electra } else if fork_name >= ForkName::Bellatrix { self.min_slashing_penalty_quotient_bellatrix diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index 5cc66214733..f0810e2bdbe 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -119,6 +119,10 @@ impl ForkName { ForkName::Electra => None, } } + + pub fn electra_enabled(self) -> bool { + self >= ForkName::Electra + } } /// Map a fork name into a fork-versioned superstruct type like `BeaconBlock`. diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index ba99a0a68d8..f2b36ee1533 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -63,7 +63,7 @@ impl Validator { spec: &ChainSpec, current_fork: ForkName, ) -> bool { - if current_fork >= ForkName::Electra { + if current_fork.electra_enabled() { self.is_eligible_for_activation_queue_electra(spec) } else { self.is_eligible_for_activation_queue_base(spec) @@ -162,7 +162,7 @@ impl Validator { spec: &ChainSpec, current_fork: ForkName, ) -> bool { - if current_fork >= ForkName::Electra { + if current_fork.electra_enabled() { self.is_fully_withdrawable_at_electra(balance, epoch, spec) } else { self.is_fully_withdrawable_at_capella(balance, epoch, spec) @@ -202,7 +202,7 @@ impl Validator { spec: &ChainSpec, current_fork: ForkName, ) -> bool { - if current_fork >= ForkName::Electra { + if current_fork.electra_enabled() { self.is_partially_withdrawable_validator_electra(balance, spec) } else { self.is_partially_withdrawable_validator_capella(balance, spec) diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index d6c8b6302d1..2a2cc067e58 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -180,7 +180,7 @@ impl LoadCase for ForkChoiceTest { }) } Step::Attestation { attestation } => { - if fork_name >= ForkName::Electra { + if fork_name.electra_enabled() { ssz_decode_file(&path.join(format!("{}.ssz_snappy", attestation))).map( |attestation| Step::Attestation { attestation: Attestation::Electra(attestation), @@ -195,7 +195,7 @@ impl LoadCase for ForkChoiceTest { } } Step::AttesterSlashing { attester_slashing } => { - if fork_name >= ForkName::Electra { + if fork_name.electra_enabled() { ssz_decode_file(&path.join(format!("{}.ssz_snappy", attester_slashing))) .map(|attester_slashing| Step::AttesterSlashing { attester_slashing: AttesterSlashing::Electra(attester_slashing), From 2634a1f1a6c45d4e75a89e884c762be4534858d9 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 18 Jun 2024 22:50:34 +0200 Subject: [PATCH 145/227] Update common/eth2/src/types.rs Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com> --- common/eth2/src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index c20e723dddb..c945e09619f 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -726,7 +726,7 @@ impl AttesterData { self.slot == attestation_data.slot && self.committee_index == attestation_data.index } else { // After electra `attestation_data.index` is set to 0 and does not match the duties - self.slot == attestation_data.index + self.slot == attestation_data.slot } } } From dec7cff9c76ddaac4f0ac0613cafafa3625bcff3 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 17 Jun 2024 22:17:48 +0200 Subject: [PATCH 146/227] Dedup attestation constructor logic in attester cache --- .../beacon_chain/src/attester_cache.rs | 2 + .../beacon_chain/src/early_attester_cache.rs | 45 +++++-------------- consensus/types/src/attestation.rs | 28 +++++++++--- validator_client/src/attestation_service.rs | 7 ++- 4 files changed, 39 insertions(+), 43 deletions(-) diff --git a/beacon_node/beacon_chain/src/attester_cache.rs b/beacon_node/beacon_chain/src/attester_cache.rs index 2e07cd32ed9..b5012e8e4e4 100644 --- a/beacon_node/beacon_chain/src/attester_cache.rs +++ b/beacon_node/beacon_chain/src/attester_cache.rs @@ -15,6 +15,7 @@ use state_processing::state_advance::{partial_state_advance, Error as StateAdvan use std::collections::HashMap; use std::ops::Range; use types::{ + attestation::Error as AttestationError, beacon_state::{ compute_committee_index_in_epoch, compute_committee_range_in_epoch, epoch_committee_count, }, @@ -59,6 +60,7 @@ pub enum Error { InverseRange { range: Range, }, + AttestationError(AttestationError), } impl From for Error { diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index 41a225d9f24..dda699cc6c1 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -6,7 +6,6 @@ use crate::{ use parking_lot::RwLock; use proto_array::Block as ProtoBlock; use std::sync::Arc; -use types::attestation::AttestationBase; use types::*; pub struct CacheItem { @@ -123,40 +122,16 @@ impl EarlyAttesterCache { item.committee_lengths .get_committee_length::(request_slot, request_index, spec)?; - let attestation = if spec.fork_name_at_slot::(request_slot).electra_enabled() { - let mut committee_bits = BitVector::default(); - if committee_len > 0 { - committee_bits - .set(request_index as usize, true) - .map_err(BeaconStateError::from)?; - } - Attestation::Electra(AttestationElectra { - aggregation_bits: BitList::with_capacity(committee_len) - .map_err(BeaconStateError::from)?, - committee_bits, - data: AttestationData { - slot: request_slot, - index: 0u64, - beacon_block_root: item.beacon_block_root, - source: item.source, - target: item.target, - }, - signature: AggregateSignature::empty(), - }) - } else { - Attestation::Base(AttestationBase { - aggregation_bits: BitList::with_capacity(committee_len) - .map_err(BeaconStateError::from)?, - data: AttestationData { - slot: request_slot, - index: request_index, - beacon_block_root: item.beacon_block_root, - source: item.source, - target: item.target, - }, - signature: AggregateSignature::empty(), - }) - }; + let attestation = Attestation::empty_for_signing( + request_index, + committee_len, + request_slot, + item.beacon_block_root, + item.source, + item.target, + spec, + ) + .map_err(Error::AttestationError)?; metrics::inc_counter(&metrics::BEACON_EARLY_ATTESTER_CACHE_HITS); diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 35bd80fc334..552c0faf6fd 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -1,5 +1,6 @@ use crate::slot_data::SlotData; use crate::{test_utils::TestRandom, Hash256, Slot}; +use crate::{Checkpoint, ForkName}; use derivative::Derivative; use safe_arith::ArithError; use serde::{Deserialize, Serialize}; @@ -89,20 +90,29 @@ impl Hash for Attestation { impl Attestation { pub fn empty_for_signing( - committee_index: usize, + committee_index: u64, committee_length: usize, - data: AttestationData, + slot: Slot, + beacon_block_root: Hash256, + source: Checkpoint, + target: Checkpoint, spec: &ChainSpec, ) -> Result { - if spec.fork_name_at_slot::(data.slot).electra_enabled() { + if spec.fork_name_at_slot::(slot) >= ForkName::Electra { let mut committee_bits: BitVector = BitVector::default(); committee_bits - .set(committee_index, true) + .set(committee_index as usize, true) .map_err(|_| Error::InvalidCommitteeIndex)?; Ok(Attestation::Electra(AttestationElectra { aggregation_bits: BitList::with_capacity(committee_length) .map_err(|_| Error::InvalidCommitteeLength)?, - data, + data: AttestationData { + slot, + index: 0u64, + beacon_block_root, + source, + target, + }, committee_bits, signature: AggregateSignature::infinity(), })) @@ -110,7 +120,13 @@ impl Attestation { Ok(Attestation::Base(AttestationBase { aggregation_bits: BitList::with_capacity(committee_length) .map_err(|_| Error::InvalidCommitteeLength)?, - data, + data: AttestationData { + slot, + index: committee_index, + beacon_block_root, + source, + target, + }, signature: AggregateSignature::infinity(), })) } diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index d08d7567afe..0cff39546dd 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -374,9 +374,12 @@ impl AttestationService { } let mut attestation = match Attestation::::empty_for_signing( - duty.committee_index as usize, + duty.committee_index, duty.committee_length as usize, - attestation_data.clone(), + attestation_data.slot, + attestation_data.beacon_block_root, + attestation_data.source, + attestation_data.target, &self.context.eth2_config.spec, ) { Ok(attestation) => attestation, From 6a4d842376eaf3839bcee126b24b996bfa877814 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 18 Jun 2024 18:00:58 +0200 Subject: [PATCH 147/227] Use if let Ok for committee_bits --- .../src/attestation_verification.rs | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 6eb0c6ecf9d..221d9114b1a 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -519,7 +519,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { .tree_hash_root(); // [New in Electra:EIP7549] - verify_committee_index(attestation, &chain.spec)?; + verify_committee_index(attestation)?; if chain .observed_attestations @@ -837,7 +837,7 @@ impl<'a, T: BeaconChainTypes> IndexedUnaggregatedAttestation<'a, T> { } // [New in Electra:EIP7549] - verify_committee_index(attestation, &chain.spec)?; + verify_committee_index(attestation)?; // Attestations must be for a known block. If the block is unknown, we simply drop the // attestation and do not delay consideration for later. @@ -1337,20 +1337,10 @@ pub fn verify_signed_aggregate_signatures( /// Verify that the `attestation` committee index is properly set for the attestation's fork. /// This function will only apply verification post-Electra. -pub fn verify_committee_index( - attestation: AttestationRef, - spec: &ChainSpec, -) -> Result<(), Error> { - if spec - .fork_name_at_slot::(attestation.data().slot) - .electra_enabled() - { +pub fn verify_committee_index(attestation: AttestationRef) -> Result<(), Error> { + if let Ok(committee_bits) = attestation.committee_bits() { // Check to ensure that the attestation is for a single committee. - let num_committee_bits = get_committee_indices::( - attestation - .committee_bits() - .map_err(|e| Error::BeaconChainError(e.into()))?, - ); + let num_committee_bits = get_committee_indices::(committee_bits); if num_committee_bits.len() != 1 { return Err(Error::NotExactlyOneCommitteeBitSet( num_committee_bits.len(), From 6f0b78426a9a4c3804159f3ac8998224645ce456 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:00:02 +0200 Subject: [PATCH 148/227] Dedup Attestation constructor code --- beacon_node/beacon_chain/src/beacon_chain.rs | 44 ++++-------------- beacon_node/beacon_chain/src/test_utils.rs | 47 +++++--------------- consensus/types/src/attestation.rs | 1 + 3 files changed, 22 insertions(+), 70 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index eeb14579548..f1d9ce791e0 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -122,7 +122,6 @@ use store::{ use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; use tree_hash::TreeHash; -use types::attestation::AttestationBase; use types::blob_sidecar::FixedBlobSidecarList; use types::payload::BlockProductionVersion; use types::*; @@ -1994,40 +1993,15 @@ impl BeaconChain { }; drop(cache_timer); - if self - .spec - .fork_name_at_slot::(request_slot) - .electra_enabled() - { - let mut committee_bits = BitVector::default(); - if committee_len > 0 { - committee_bits.set(request_index as usize, true)?; - } - Ok(Attestation::Electra(AttestationElectra { - aggregation_bits: BitList::with_capacity(committee_len)?, - data: AttestationData { - slot: request_slot, - index: 0u64, - beacon_block_root, - source: justified_checkpoint, - target, - }, - committee_bits, - signature: AggregateSignature::empty(), - })) - } else { - Ok(Attestation::Base(AttestationBase { - aggregation_bits: BitList::with_capacity(committee_len)?, - data: AttestationData { - slot: request_slot, - index: request_index, - beacon_block_root, - source: justified_checkpoint, - target, - }, - signature: AggregateSignature::empty(), - })) - } + Ok(Attestation::::empty_for_signing( + request_index, + committee_len, + request_slot, + beacon_block_root, + justified_checkpoint, + target, + &self.spec, + )?) } /// Performs the same validation as `Self::verify_unaggregated_attestation_for_gossip`, but for diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 7d443db3809..5a37c5e6cee 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -58,7 +58,6 @@ use store::{config::StoreConfig, HotColdDB, ItemStore, LevelDB, MemoryStore}; use task_executor::TaskExecutor; use task_executor::{test_utils::TestRuntime, ShutdownReason}; use tree_hash::TreeHash; -use types::attestation::AttestationBase; use types::indexed_attestation::IndexedAttestationBase; use types::payload::BlockProductionVersion; pub use types::test_utils::generate_deterministic_keypairs; @@ -1033,40 +1032,18 @@ where *state.get_block_root(target_slot)? }; - if self.spec.fork_name_at_slot::(slot).electra_enabled() { - let mut committee_bits = BitVector::default(); - committee_bits.set(index as usize, true)?; - Ok(Attestation::Electra(AttestationElectra { - aggregation_bits: BitList::with_capacity(committee_len)?, - committee_bits, - data: AttestationData { - slot, - index: 0u64, - beacon_block_root, - source: state.current_justified_checkpoint(), - target: Checkpoint { - epoch, - root: target_root, - }, - }, - signature: AggregateSignature::empty(), - })) - } else { - Ok(Attestation::Base(AttestationBase { - aggregation_bits: BitList::with_capacity(committee_len)?, - data: AttestationData { - slot, - index, - beacon_block_root, - source: state.current_justified_checkpoint(), - target: Checkpoint { - epoch, - root: target_root, - }, - }, - signature: AggregateSignature::empty(), - })) - } + Ok(Attestation::empty_for_signing( + index, + committee_len, + slot, + beacon_block_root, + state.current_justified_checkpoint(), + Checkpoint { + epoch, + root: target_root, + }, + &self.spec, + )?) } /// A list of attestations for each committee for the given slot. diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 552c0faf6fd..c8d1c3fb9b7 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -89,6 +89,7 @@ impl Hash for Attestation { } impl Attestation { + /// Produces an attestation with empty signature. pub fn empty_for_signing( committee_index: u64, committee_length: usize, From 444cd625ef3aa9ca732334568ef8d54b7024c2e6 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:11:09 +0200 Subject: [PATCH 149/227] Diff reduction in tests --- beacon_node/beacon_chain/src/test_utils.rs | 84 +++++++------------ .../tests/attestation_production.rs | 27 ++---- 2 files changed, 38 insertions(+), 73 deletions(-) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 5a37c5e6cee..bd98f19af6f 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1354,36 +1354,25 @@ where let fork_name = self.spec.fork_name_at_slot::(slot); let aggregate = if fork_name.electra_enabled() { - self.chain - .get_aggregated_attestation_electra( - slot, - &attestation.data().tree_hash_root(), - bc.index, - ) - .unwrap() - .unwrap_or_else(|| { - committee_attestations.iter().skip(1).fold( - attestation.clone(), - |mut agg, (att, _)| { - agg.aggregate(att.to_ref()); - agg - }, - ) - }) + self.chain.get_aggregated_attestation_electra( + slot, + &attestation.data().tree_hash_root(), + bc.index, + ) } else { self.chain .get_aggregated_attestation_base(attestation.data()) - .unwrap() - .unwrap_or_else(|| { - committee_attestations.iter().skip(1).fold( - attestation.clone(), - |mut agg, (att, _)| { - agg.aggregate(att.to_ref()); - agg - }, - ) - }) - }; + } + .unwrap() + .unwrap_or_else(|| { + committee_attestations.iter().skip(1).fold( + attestation.clone(), + |mut agg, (att, _)| { + agg.aggregate(att.to_ref()); + agg + }, + ) + }); // If the chain is able to produce an aggregate, use that. Otherwise, build an // aggregate locally. @@ -1518,40 +1507,29 @@ where let fork_name = self.spec.fork_name_at_slot::(Slot::new(0)); + let data = AttestationData { + slot: Slot::new(0), + index: 0, + beacon_block_root: Hash256::zero(), + target: Checkpoint { + root: Hash256::zero(), + epoch: target1.unwrap_or(fork.epoch), + }, + source: Checkpoint { + root: Hash256::zero(), + epoch: source1.unwrap_or(Epoch::new(0)), + }, + }; let mut attestation_1 = if fork_name.electra_enabled() { IndexedAttestation::Electra(IndexedAttestationElectra { attesting_indices: VariableList::new(validator_indices).unwrap(), - data: AttestationData { - slot: Slot::new(0), - index: 0, - beacon_block_root: Hash256::zero(), - target: Checkpoint { - root: Hash256::zero(), - epoch: target1.unwrap_or(fork.epoch), - }, - source: Checkpoint { - root: Hash256::zero(), - epoch: source1.unwrap_or(Epoch::new(0)), - }, - }, + data, signature: AggregateSignature::infinity(), }) } else { IndexedAttestation::Base(IndexedAttestationBase { attesting_indices: VariableList::new(validator_indices).unwrap(), - data: AttestationData { - slot: Slot::new(0), - index: 0, - beacon_block_root: Hash256::zero(), - target: Checkpoint { - root: Hash256::zero(), - epoch: target1.unwrap_or(fork.epoch), - }, - source: Checkpoint { - root: Hash256::zero(), - epoch: source1.unwrap_or(Epoch::new(0)), - }, - }, + data, signature: AggregateSignature::infinity(), }) }; diff --git a/beacon_node/beacon_chain/tests/attestation_production.rs b/beacon_node/beacon_chain/tests/attestation_production.rs index a58cfd9f2d1..1716cd262fd 100644 --- a/beacon_node/beacon_chain/tests/attestation_production.rs +++ b/beacon_node/beacon_chain/tests/attestation_production.rs @@ -190,30 +190,17 @@ async fn produces_attestations() { .produce_unaggregated_attestation(slot, index) .expect("should produce attestation"); - match &attestation { + let (aggregation_bits_len, aggregation_bits_zero) = match &attestation { Attestation::Base(att) => { - assert_eq!( - att.aggregation_bits.len(), - committee_len, - "bad committee len" - ); - assert!( - att.aggregation_bits.is_zero(), - "some committee bits are set" - ); + (att.aggregation_bits.len(), att.aggregation_bits.is_zero()) } Attestation::Electra(att) => { - assert_eq!( - att.aggregation_bits.len(), - committee_len, - "bad committee len" - ); - assert!( - att.aggregation_bits.is_zero(), - "some committee bits are set" - ); + (att.aggregation_bits.len(), att.aggregation_bits.is_zero()) } - } + }; + assert_eq!(aggregation_bits_len, committee_len, "bad committee len"); + assert!(aggregation_bits_zero, "some committee bits are set"); + let data = attestation.data(); assert_eq!( From d26473621a4392b1e4efc354c44e9828b9c2d664 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:38:25 +0200 Subject: [PATCH 150/227] Fix beacon_chain tests --- beacon_node/beacon_chain/src/observed_aggregates.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/observed_aggregates.rs b/beacon_node/beacon_chain/src/observed_aggregates.rs index b02bff96a2d..00476bfe7af 100644 --- a/beacon_node/beacon_chain/src/observed_aggregates.rs +++ b/beacon_node/beacon_chain/src/observed_aggregates.rs @@ -473,12 +473,13 @@ where #[cfg(not(debug_assertions))] mod tests { use super::*; - use types::{test_utils::test_random_instance, Hash256}; + use types::{test_utils::test_random_instance, AttestationBase, Hash256}; type E = types::MainnetEthSpec; fn get_attestation(slot: Slot, beacon_block_root: u64) -> Attestation { - let mut a: Attestation = test_random_instance(); + let a: AttestationBase = test_random_instance(); + let mut a = Attestation::Base(a); a.data_mut().slot = slot; a.data_mut().beacon_block_root = Hash256::from_low_u64_be(beacon_block_root); a From 7521f97ca5fcfd63b283d6cba0b0d0ffd548a5e9 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:38:35 +0200 Subject: [PATCH 151/227] Diff reduction --- .../beacon_chain/tests/block_verification.rs | 84 +++++++------------ 1 file changed, 28 insertions(+), 56 deletions(-) diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 55b2c14a3c5..4c7a499697d 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -1308,62 +1308,34 @@ async fn verify_block_for_gossip_doppelganger_detection() { } }; - match indexed_attestation { - IndexedAttestation::Base(indexed_attestation) => { - for &index in &indexed_attestation.attesting_indices { - let index = index as usize; - - assert!(harness.chain.validator_seen_at_epoch(index, epoch)); - - // Check the correct beacon cache is populated - assert!(harness - .chain - .observed_block_attesters - .read() - .validator_has_been_observed(epoch, index) - .expect("should check if block attester was observed")); - assert!(!harness - .chain - .observed_gossip_attesters - .read() - .validator_has_been_observed(epoch, index) - .expect("should check if gossip attester was observed")); - assert!(!harness - .chain - .observed_aggregators - .read() - .validator_has_been_observed(epoch, index) - .expect("should check if gossip aggregator was observed")); - } - } - IndexedAttestation::Electra(indexed_attestation) => { - for &index in &indexed_attestation.attesting_indices { - let index = index as usize; - - assert!(harness.chain.validator_seen_at_epoch(index, epoch)); - - // Check the correct beacon cache is populated - assert!(harness - .chain - .observed_block_attesters - .read() - .validator_has_been_observed(epoch, index) - .expect("should check if block attester was observed")); - assert!(!harness - .chain - .observed_gossip_attesters - .read() - .validator_has_been_observed(epoch, index) - .expect("should check if gossip attester was observed")); - assert!(!harness - .chain - .observed_aggregators - .read() - .validator_has_been_observed(epoch, index) - .expect("should check if gossip aggregator was observed")); - } - } - }; + for index in match indexed_attestation { + IndexedAttestation::Base(att) => att.attesting_indices.into_iter(), + IndexedAttestation::Electra(att) => att.attesting_indices.into_iter(), + } { + let index = index as usize; + + assert!(harness.chain.validator_seen_at_epoch(index, epoch)); + + // Check the correct beacon cache is populated + assert!(harness + .chain + .observed_block_attesters + .read() + .validator_has_been_observed(epoch, index) + .expect("should check if block attester was observed")); + assert!(!harness + .chain + .observed_gossip_attesters + .read() + .validator_has_been_observed(epoch, index) + .expect("should check if gossip attester was observed")); + assert!(!harness + .chain + .observed_aggregators + .read() + .validator_has_been_observed(epoch, index) + .expect("should check if gossip aggregator was observed")); + } } } From 4d3edfeaedb2e43314d1953dbfc7b82ef4133f73 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:46:12 +0200 Subject: [PATCH 152/227] Use Ord for ForkName in pubsub --- .../lighthouse_network/src/types/pubsub.rs | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 7be68f879fe..c7b9ee0ec99 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -158,18 +158,19 @@ impl PubsubMessage { GossipKind::BeaconAggregateAndProof => { let signed_aggregate_and_proof = match fork_context.from_context_bytes(gossip_topic.fork_digest) { - Some(ForkName::Base) - | Some(ForkName::Altair) - | Some(ForkName::Bellatrix) - | Some(ForkName::Capella) - | Some(ForkName::Deneb) => SignedAggregateAndProof::Base( - SignedAggregateAndProofBase::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, - ), - Some(ForkName::Electra) => SignedAggregateAndProof::Electra( - SignedAggregateAndProofElectra::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, - ), + Some(&fork_name) => { + if fork_name >= ForkName::Electra { + SignedAggregateAndProof::Electra( + SignedAggregateAndProofElectra::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ) + } else { + SignedAggregateAndProof::Base( + SignedAggregateAndProofBase::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ) + } + } None => { return Err(format!( "Unknown gossipsub fork digest: {:?}", @@ -184,18 +185,19 @@ impl PubsubMessage { GossipKind::Attestation(subnet_id) => { let attestation = match fork_context.from_context_bytes(gossip_topic.fork_digest) { - Some(ForkName::Base) - | Some(ForkName::Altair) - | Some(ForkName::Bellatrix) - | Some(ForkName::Capella) - | Some(ForkName::Deneb) => Attestation::Base( - AttestationBase::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, - ), - Some(ForkName::Electra) => Attestation::Electra( - AttestationElectra::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, - ), + Some(&fork_name) => { + if fork_name >= ForkName::Electra { + Attestation::Electra( + AttestationElectra::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ) + } else { + Attestation::Base( + AttestationBase::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ) + } + } None => { return Err(format!( "Unknown gossipsub fork digest: {:?}", @@ -281,18 +283,19 @@ impl PubsubMessage { GossipKind::AttesterSlashing => { let attester_slashing = match fork_context.from_context_bytes(gossip_topic.fork_digest) { - Some(ForkName::Base) - | Some(ForkName::Altair) - | Some(ForkName::Bellatrix) - | Some(ForkName::Capella) - | Some(ForkName::Deneb) => AttesterSlashing::Base( - AttesterSlashingBase::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, - ), - Some(ForkName::Electra) => AttesterSlashing::Electra( - AttesterSlashingElectra::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, - ), + Some(&fork_name) => { + if fork_name >= ForkName::Electra { + AttesterSlashing::Electra( + AttesterSlashingElectra::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ) + } else { + AttesterSlashing::Base( + AttesterSlashingBase::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ) + } + } None => { return Err(format!( "Unknown gossipsub fork digest: {:?}", From 7fce143300a6310677b974ef729a482a467f8e27 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 18 Jun 2024 22:16:37 +0200 Subject: [PATCH 153/227] Resolve into_attestation_and_indices todo --- .../src/network_beacon_processor/gossip_methods.rs | 7 +------ consensus/types/src/signed_aggregate_and_proof.rs | 8 ++++++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 34eb8b26b13..8cbc7782188 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -105,12 +105,7 @@ impl VerifiedAttestation for VerifiedAggregate { /// Efficient clone-free implementation that moves out of the `Box`. fn into_attestation_and_indices(self) -> (Attestation, Vec) { - // TODO(electra): technically we shouldn't have to clone.. - let attestation = self - .signed_aggregate - .message() - .aggregate() - .clone_as_attestation(); + let attestation = self.signed_aggregate.into_attestation(); let attesting_indices = self.indexed_attestation.attesting_indices_to_vec(); (attestation, attesting_indices) } diff --git a/consensus/types/src/signed_aggregate_and_proof.rs b/consensus/types/src/signed_aggregate_and_proof.rs index 94624f02fe7..d339cecaae0 100644 --- a/consensus/types/src/signed_aggregate_and_proof.rs +++ b/consensus/types/src/signed_aggregate_and_proof.rs @@ -6,6 +6,7 @@ use super::{ Signature, SignedRoot, }; use crate::test_utils::TestRandom; +use crate::Attestation; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; @@ -109,4 +110,11 @@ impl SignedAggregateAndProof { } } } + + pub fn into_attestation(self) -> Attestation { + match self { + Self::Base(att) => Attestation::Base(att.message.aggregate), + Self::Electra(att) => Attestation::Electra(att.message.aggregate), + } + } } From 4d4c268e1eff67a09fa1d4a6c29f6c55a670adbd Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 18 Jun 2024 22:29:50 +0200 Subject: [PATCH 154/227] Remove stale TODO --- beacon_node/operation_pool/src/attestation_storage.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index 43b1c3abbb3..ad4ffe6d3c1 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -217,7 +217,6 @@ impl CompactIndexedAttestationElectra { } pub fn aggregate_same_committee(&mut self, other: &Self) -> Option<()> { - // TODO(electra): remove assert in favour of Result if self.committee_bits != other.committee_bits { return None; } From 370d5112232d58b5eb793d2c4553d66f82f7c6fc Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 18 Jun 2024 22:52:40 +0200 Subject: [PATCH 155/227] Fix beacon_chain tests --- beacon_node/beacon_chain/tests/attestation_production.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/tests/attestation_production.rs b/beacon_node/beacon_chain/tests/attestation_production.rs index 1716cd262fd..697e449dc6e 100644 --- a/beacon_node/beacon_chain/tests/attestation_production.rs +++ b/beacon_node/beacon_chain/tests/attestation_production.rs @@ -205,7 +205,7 @@ async fn produces_attestations() { assert_eq!( attestation.signature(), - &AggregateSignature::empty(), + &AggregateSignature::infinity(), "bad signature" ); assert_eq!(data.index, index, "bad index"); From cbb7c5d8f41c10d99aa84653c665ae5cc7a5f843 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 19 Jun 2024 11:39:45 +0200 Subject: [PATCH 156/227] Test spec invariant --- consensus/types/src/eth_spec.rs | 2 ++ consensus/types/src/indexed_attestation.rs | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index cdbab2d5406..1c379f5de42 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -545,10 +545,12 @@ impl EthSpec for GnosisEthSpec { #[cfg(test)] mod test { use crate::{EthSpec, GnosisEthSpec, MainnetEthSpec, MinimalEthSpec}; + use ssz_types::typenum::Unsigned; fn assert_valid_spec() { E::kzg_commitments_tree_depth(); E::block_body_tree_depth(); + assert!(E::MaxValidatorsPerSlot::to_i32() >= E::MaxValidatorsPerCommittee::to_i32()); } #[test] diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index 429d65898a9..900c9059389 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -120,7 +120,8 @@ impl IndexedAttestation { let extended_attesting_indices: VariableList = VariableList::new(att.attesting_indices.to_vec()) .expect("MaxValidatorsPerSlot must be >= MaxValidatorsPerCommittee"); - // TODO: Add test after unstable rebase https://github.com/sigp/lighthouse/blob/474c1b44863927c588dd05ab2ac0f934298398e1/consensus/types/src/eth_spec.rs#L541 + // Note a unit test in consensus/types/src/eth_spec.rs asserts this invariant for + // all known specs IndexedAttestationElectra { attesting_indices: extended_attesting_indices, From 70a2d4de1055496edd4d1d4267f9b01cfd0c0cd8 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 19 Jun 2024 11:43:41 +0200 Subject: [PATCH 157/227] Use electra_enabled in pubsub --- beacon_node/lighthouse_network/src/types/pubsub.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index c7b9ee0ec99..b443ecd1b9b 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -159,7 +159,7 @@ impl PubsubMessage { let signed_aggregate_and_proof = match fork_context.from_context_bytes(gossip_topic.fork_digest) { Some(&fork_name) => { - if fork_name >= ForkName::Electra { + if fork_name.electra_enabled() { SignedAggregateAndProof::Electra( SignedAggregateAndProofElectra::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, @@ -186,7 +186,7 @@ impl PubsubMessage { let attestation = match fork_context.from_context_bytes(gossip_topic.fork_digest) { Some(&fork_name) => { - if fork_name >= ForkName::Electra { + if fork_name.electra_enabled() { Attestation::Electra( AttestationElectra::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, @@ -284,7 +284,7 @@ impl PubsubMessage { let attester_slashing = match fork_context.from_context_bytes(gossip_topic.fork_digest) { Some(&fork_name) => { - if fork_name >= ForkName::Electra { + if fork_name.electra_enabled() { AttesterSlashing::Electra( AttesterSlashingElectra::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, From 9e6e76fb898ad9349a2bd383fbf8543ffc915299 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:47:38 +0200 Subject: [PATCH 158/227] Remove get_indexed_attestation_from_signed_aggregate --- .../src/attestation_verification.rs | 99 +++++++++-------- .../src/common/get_attesting_indices.rs | 103 ++---------------- 2 files changed, 60 insertions(+), 142 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 221d9114b1a..a5635a3392c 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -61,9 +61,10 @@ use std::borrow::Cow; use strum::AsRefStr; use tree_hash::TreeHash; use types::{ - Attestation, AttestationRef, BeaconCommittee, BeaconStateError, - BeaconStateError::NoCommitteeFound, ChainSpec, CommitteeIndex, Epoch, EthSpec, ForkName, - Hash256, IndexedAttestation, SelectionProof, SignedAggregateAndProof, Slot, SubnetId, + Attestation, AttestationRef, BeaconCommittee, + BeaconStateError::{self, NoCommitteeFound}, + ChainSpec, CommitteeIndex, Epoch, EthSpec, ForkName, Hash256, IndexedAttestation, + SelectionProof, SignedAggregateAndProof, Slot, SubnetId, }; pub use batch::{batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations}; @@ -598,57 +599,61 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { }; let get_indexed_attestation_with_committee = |(committees, _): (Vec, CommitteesPerSlot)| { - match signed_aggregate { - SignedAggregateAndProof::Base(signed_aggregate) => { - let att = &signed_aggregate.message.aggregate; - let aggregator_index = signed_aggregate.message.aggregator_index; - let committee = committees - .iter() - .filter(|&committee| committee.index == att.data.index) - .at_most_one() - .map_err(|_| Error::NoCommitteeForSlotAndIndex { - slot: att.data.slot, - index: att.data.index, - })?; - - // TODO(electra): + let (index, aggregator_index, selection_proof, data) = match signed_aggregate { + SignedAggregateAndProof::Base(signed_aggregate) => ( + signed_aggregate.message.aggregate.data.index, + signed_aggregate.message.aggregator_index, // Note: this clones the signature which is known to be a relatively slow operation. - // // Future optimizations should remove this clone. - if let Some(committee) = committee { - let selection_proof = SelectionProof::from( - signed_aggregate.message.selection_proof.clone(), - ); - - if !selection_proof - .is_aggregator(committee.committee.len(), &chain.spec) - .map_err(|e| Error::BeaconChainError(e.into()))? - { - return Err(Error::InvalidSelectionProof { aggregator_index }); - } + signed_aggregate.message.selection_proof.clone(), + signed_aggregate.message.aggregate.data.clone(), + ), + SignedAggregateAndProof::Electra(signed_aggregate) => ( + signed_aggregate + .message + .aggregate + .committee_index() + .ok_or(Error::NotExactlyOneCommitteeBitSet(0))?, + signed_aggregate.message.aggregator_index, + signed_aggregate.message.selection_proof.clone(), + signed_aggregate.message.aggregate.data.clone(), + ), + }; + let slot = data.slot; - // Ensure the aggregator is a member of the committee for which it is aggregating. - if !committee.committee.contains(&(aggregator_index as usize)) { - return Err(Error::AggregatorNotInCommittee { aggregator_index }); - } + let committee = committees + .iter() + .filter(|&committee| committee.index == index) + .at_most_one() + .map_err(|_| Error::NoCommitteeForSlotAndIndex { slot, index })? + .ok_or(Error::NoCommitteeForSlotAndIndex { slot, index })?; - attesting_indices_base::get_indexed_attestation( - committee.committee, - att, - ) - .map_err(|e| BeaconChainError::from(e).into()) - } else { - Err(Error::NoCommitteeForSlotAndIndex { - slot: att.data.slot, - index: att.data.index, - }) - } + if !SelectionProof::from(selection_proof) + .is_aggregator(committee.committee.len(), &chain.spec) + .map_err(|e| Error::BeaconChainError(e.into()))? + { + return Err(Error::InvalidSelectionProof { aggregator_index }); + } + + // Ensure the aggregator is a member of the committee for which it is aggregating. + if !committee.committee.contains(&(aggregator_index as usize)) { + return Err(Error::AggregatorNotInCommittee { aggregator_index }); + } + + // p2p aggregates have a single committee, we can assert that aggregation_bits is always + // less then MaxValidatorsPerCommittee + match signed_aggregate { + SignedAggregateAndProof::Base(signed_aggregate) => { + attesting_indices_base::get_indexed_attestation( + committee.committee, + &signed_aggregate.message.aggregate, + ) + .map_err(|e| BeaconChainError::from(e).into()) } SignedAggregateAndProof::Electra(signed_aggregate) => { - attesting_indices_electra::get_indexed_attestation_from_signed_aggregate( + attesting_indices_electra::get_indexed_attestation_from_committees( &committees, - signed_aggregate, - &chain.spec, + &signed_aggregate.message.aggregate, ) .map_err(|e| BeaconChainError::from(e).into()) } diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index a79604003ee..f24b003665d 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -51,100 +51,6 @@ pub mod attesting_indices_electra { use safe_arith::SafeArith; use types::*; - // TODO(electra) remove duplicate code - // get_indexed_attestation is almost an exact duplicate - // the only differences are the invalid selection proof - // and aggregator not in committee checks - pub fn get_indexed_attestation_from_signed_aggregate( - committees: &[BeaconCommittee], - signed_aggregate: &SignedAggregateAndProofElectra, - spec: &ChainSpec, - ) -> Result, BeaconStateError> { - let mut output: HashSet = HashSet::new(); - - let committee_bits = &signed_aggregate.message.aggregate.committee_bits; - let aggregation_bits = &signed_aggregate.message.aggregate.aggregation_bits; - let aggregator_index = signed_aggregate.message.aggregator_index; - let attestation = &signed_aggregate.message.aggregate; - - let committee_indices = get_committee_indices::(committee_bits); - - let mut committee_offset = 0; - - let committees_map: HashMap = committees - .iter() - .map(|committee| (committee.index, committee)) - .collect(); - - let committee_count_per_slot = committees.len() as u64; - let mut participant_count = 0; - - // TODO(electra): - // Note: this clones the signature which is known to be a relatively slow operation. - // - // Future optimizations should remove this clone. - let selection_proof = - SelectionProof::from(signed_aggregate.message.selection_proof.clone()); - - for index in committee_indices { - if let Some(&beacon_committee) = committees_map.get(&index) { - if !selection_proof - .is_aggregator(beacon_committee.committee.len(), spec) - .map_err(BeaconStateError::ArithError)? - { - return Err(BeaconStateError::InvalidSelectionProof { aggregator_index }); - } - - if !beacon_committee - .committee - .contains(&(aggregator_index as usize)) - { - return Err(BeaconStateError::AggregatorNotInCommittee { aggregator_index }); - } - - // This check is new to the spec's `process_attestation` in Electra. - if index >= committee_count_per_slot { - return Err(BeaconStateError::InvalidCommitteeIndex(index)); - } - - participant_count.safe_add_assign(beacon_committee.committee.len() as u64)?; - let committee_attesters = beacon_committee - .committee - .iter() - .enumerate() - .filter_map(|(i, &index)| { - if let Ok(aggregation_bit_index) = committee_offset.safe_add(i) { - if aggregation_bits.get(aggregation_bit_index).unwrap_or(false) { - return Some(index as u64); - } - } - None - }) - .collect::>(); - - output.extend(committee_attesters); - - committee_offset.safe_add_assign(beacon_committee.committee.len())?; - } else { - return Err(Error::NoCommitteeFound(index)); - } - } - - // This check is new to the spec's `process_attestation` in Electra. - if participant_count as usize != aggregation_bits.len() { - return Err(Error::InvalidBitfield); - } - - let mut indices = output.into_iter().collect_vec(); - indices.sort_unstable(); - - Ok(IndexedAttestation::Electra(IndexedAttestationElectra { - attesting_indices: VariableList::new(indices)?, - data: attestation.data.clone(), - signature: attestation.signature.clone(), - })) - } - pub fn get_indexed_attestation( committees: &[BeaconCommittee], attestation: &AttestationElectra, @@ -167,8 +73,15 @@ pub mod attesting_indices_electra { attestation: &AttestationElectra, ) -> Result, BlockOperationError> { let committees = beacon_state.get_beacon_committees_at_slot(attestation.data.slot)?; + get_indexed_attestation_from_committees(&committees, attestation) + } + + pub fn get_indexed_attestation_from_committees( + committees: &[BeaconCommittee], + attestation: &AttestationElectra, + ) -> Result, BlockOperationError> { let attesting_indices = get_attesting_indices::( - &committees, + committees, &attestation.aggregation_bits, &attestation.committee_bits, )?; From a8d8989c0507ed8246cdf9f657d1adb141772d79 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:50:41 +0200 Subject: [PATCH 159/227] Use ok_or instead of if let else --- .../src/common/get_attesting_indices.rs | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index f24b003665d..9fb88407d30 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -122,32 +122,32 @@ pub mod attesting_indices_electra { let committee_count_per_slot = committees.len() as u64; let mut participant_count = 0; for index in committee_indices { - if let Some(&beacon_committee) = committees_map.get(&index) { - // This check is new to the spec's `process_attestation` in Electra. - if index >= committee_count_per_slot { - return Err(BeaconStateError::InvalidCommitteeIndex(index)); - } - participant_count.safe_add_assign(beacon_committee.committee.len() as u64)?; - let committee_attesters = beacon_committee - .committee - .iter() - .enumerate() - .filter_map(|(i, &index)| { - if let Ok(aggregation_bit_index) = committee_offset.safe_add(i) { - if aggregation_bits.get(aggregation_bit_index).unwrap_or(false) { - return Some(index as u64); - } + let beacon_committee = committees_map + .get(&index) + .ok_or(Error::NoCommitteeFound(index))?; + + // This check is new to the spec's `process_attestation` in Electra. + if index >= committee_count_per_slot { + return Err(BeaconStateError::InvalidCommitteeIndex(index)); + } + participant_count.safe_add_assign(beacon_committee.committee.len() as u64)?; + let committee_attesters = beacon_committee + .committee + .iter() + .enumerate() + .filter_map(|(i, &index)| { + if let Ok(aggregation_bit_index) = committee_offset.safe_add(i) { + if aggregation_bits.get(aggregation_bit_index).unwrap_or(false) { + return Some(index as u64); } - None - }) - .collect::>(); + } + None + }) + .collect::>(); - output.extend(committee_attesters); + output.extend(committee_attesters); - committee_offset.safe_add_assign(beacon_committee.committee.len())?; - } else { - return Err(Error::NoCommitteeFound(index)); - } + committee_offset.safe_add_assign(beacon_committee.committee.len())?; } // This check is new to the spec's `process_attestation` in Electra. From d67270f8992569c59223fc26141b5a2e575c74b9 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:59:27 +0200 Subject: [PATCH 160/227] committees are sorted --- .../beacon_chain/src/attestation_verification.rs | 9 +++++---- .../src/common/get_attesting_indices.rs | 16 ++++++++-------- .../types/src/beacon_state/committee_cache.rs | 2 ++ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index a5635a3392c..51b5b11b09a 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -597,6 +597,8 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { )) } }; + + // Committees must be sorted by ascending index order 0..committees_per_slot let get_indexed_attestation_with_committee = |(committees, _): (Vec, CommitteesPerSlot)| { let (index, aggregator_index, selection_proof, data) = match signed_aggregate { @@ -622,10 +624,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { let slot = data.slot; let committee = committees - .iter() - .filter(|&committee| committee.index == index) - .at_most_one() - .map_err(|_| Error::NoCommitteeForSlotAndIndex { slot, index })? + .get(index as usize) .ok_or(Error::NoCommitteeForSlotAndIndex { slot, index })?; if !SelectionProof::from(selection_proof) @@ -1422,6 +1421,8 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( /// /// If the committees for an `attestation`'s slot isn't found in the `shuffling_cache`, we will read a state /// from disk and then update the `shuffling_cache`. +/// +/// Committees are sorted by ascending index order 0..committees_per_slot fn map_attestation_committees( chain: &BeaconChain, attestation: AttestationRef, diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 9fb88407d30..06571009995 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -44,13 +44,16 @@ pub mod attesting_indices_base { } pub mod attesting_indices_electra { - use std::collections::{HashMap, HashSet}; + use std::collections::HashSet; use crate::per_block_processing::errors::{AttestationInvalid as Invalid, BlockOperationError}; use itertools::Itertools; use safe_arith::SafeArith; use types::*; + /// Compute an Electra IndexedAttestation given a list of committees. + /// + /// Committees must be sorted by ascending order 0..committees_per_slot pub fn get_indexed_attestation( committees: &[BeaconCommittee], attestation: &AttestationElectra, @@ -103,6 +106,8 @@ pub mod attesting_indices_electra { } /// Returns validator indices which participated in the attestation, sorted by increasing index. + /// + /// Committees must be sorted by ascending order 0..committees_per_slot pub fn get_attesting_indices( committees: &[BeaconCommittee], aggregation_bits: &BitList, @@ -114,16 +119,11 @@ pub mod attesting_indices_electra { let mut committee_offset = 0; - let committees_map: HashMap = committees - .iter() - .map(|committee| (committee.index, committee)) - .collect(); - let committee_count_per_slot = committees.len() as u64; let mut participant_count = 0; for index in committee_indices { - let beacon_committee = committees_map - .get(&index) + let beacon_committee = committees + .get(index as usize) .ok_or(Error::NoCommitteeFound(index))?; // This check is new to the spec's `process_attestation` in Electra. diff --git a/consensus/types/src/beacon_state/committee_cache.rs b/consensus/types/src/beacon_state/committee_cache.rs index 209659ea889..161f8541573 100644 --- a/consensus/types/src/beacon_state/committee_cache.rs +++ b/consensus/types/src/beacon_state/committee_cache.rs @@ -183,6 +183,8 @@ impl CommitteeCache { } /// Get all the Beacon committees at a given `slot`. + /// + /// Committees are sorted by ascending index order 0..committees_per_slot pub fn get_beacon_committees_at_slot(&self, slot: Slot) -> Result, Error> { if self.initialized_epoch.is_none() { return Err(Error::CommitteeCacheUninitialized(None)); From 3977b92c49047087d8b93a46dd8d3229d2214233 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 19 Jun 2024 13:45:47 -0400 Subject: [PATCH 161/227] remove dup method `get_indexed_attestation_from_committees` --- .../src/attestation_verification.rs | 2 +- .../src/common/get_attesting_indices.rs | 19 +------------------ 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 51b5b11b09a..6044cafd6e0 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -650,7 +650,7 @@ impl<'a, T: BeaconChainTypes> IndexedAggregatedAttestation<'a, T> { .map_err(|e| BeaconChainError::from(e).into()) } SignedAggregateAndProof::Electra(signed_aggregate) => { - attesting_indices_electra::get_indexed_attestation_from_committees( + attesting_indices_electra::get_indexed_attestation( &committees, &signed_aggregate.message.aggregate, ) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 06571009995..457a5261041 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -76,24 +76,7 @@ pub mod attesting_indices_electra { attestation: &AttestationElectra, ) -> Result, BlockOperationError> { let committees = beacon_state.get_beacon_committees_at_slot(attestation.data.slot)?; - get_indexed_attestation_from_committees(&committees, attestation) - } - - pub fn get_indexed_attestation_from_committees( - committees: &[BeaconCommittee], - attestation: &AttestationElectra, - ) -> Result, BlockOperationError> { - let attesting_indices = get_attesting_indices::( - committees, - &attestation.aggregation_bits, - &attestation.committee_bits, - )?; - - Ok(IndexedAttestation::Electra(IndexedAttestationElectra { - attesting_indices: VariableList::new(attesting_indices)?, - data: attestation.data.clone(), - signature: attestation.signature.clone(), - })) + get_indexed_attestation(&committees, attestation) } /// Shortcut for getting the attesting indices while fetching the committee from the state's cache. From afb9122cc1c9b243542dddaaac611493c01f7af6 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 19 Jun 2024 15:00:33 -0400 Subject: [PATCH 162/227] update default persisted op pool deserialization --- beacon_node/operation_pool/src/persistence.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/operation_pool/src/persistence.rs b/beacon_node/operation_pool/src/persistence.rs index 99cb1aafbc5..79509e5f6cc 100644 --- a/beacon_node/operation_pool/src/persistence.rs +++ b/beacon_node/operation_pool/src/persistence.rs @@ -258,8 +258,8 @@ impl StoreItem for PersistedOperationPool { fn from_store_bytes(bytes: &[u8]) -> Result { // Default deserialization to the latest variant. - PersistedOperationPoolV15::from_ssz_bytes(bytes) - .map(Self::V15) + PersistedOperationPoolV20::from_ssz_bytes(bytes) + .map(Self::V20) .map_err(Into::into) } } From 381bbaba94df0ba23a267f92038f4637942fd2d3 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 19 Jun 2024 17:04:47 -0400 Subject: [PATCH 163/227] ensure aggregate and proof uses serde untagged on ref --- consensus/types/src/aggregate_and_proof.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/types/src/aggregate_and_proof.rs b/consensus/types/src/aggregate_and_proof.rs index ae80369130a..6ae0df4ad7c 100644 --- a/consensus/types/src/aggregate_and_proof.rs +++ b/consensus/types/src/aggregate_and_proof.rs @@ -31,7 +31,7 @@ use tree_hash_derive::TreeHash; ), ref_attributes( derive(Debug, PartialEq, TreeHash, Serialize,), - serde(bound = "E: EthSpec"), + serde(untagged, bound = "E: EthSpec"), tree_hash(enum_behaviour = "transparent") ) )] From 0e2add2daad556a771a9d66130bf277a8c86968b Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 20 Jun 2024 09:58:53 +0200 Subject: [PATCH 164/227] Fork aware ssz static attestation tests --- testing/ef_tests/tests/tests.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index fb8bdfcae71..9e8b375b2c1 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -219,7 +219,6 @@ mod ssz_static { use types::historical_summary::HistoricalSummary; use types::{AttesterSlashingBase, AttesterSlashingElectra, LightClientBootstrapAltair, *}; - ssz_static_test!(attestation, Attestation<_>); ssz_static_test!(attestation_data, AttestationData); ssz_static_test!(beacon_block, SszStaticWithSpecHandler, BeaconBlock<_>); ssz_static_test!(beacon_block_header, BeaconBlockHeader); @@ -247,6 +246,16 @@ mod ssz_static { ssz_static_test!(validator, Validator); ssz_static_test!(voluntary_exit, VoluntaryExit); + #[test] + fn attestation() { + SszStaticHandler::, MinimalEthSpec>::pre_electra().run(); + SszStaticHandler::, MainnetEthSpec>::pre_electra().run(); + SszStaticHandler::, MinimalEthSpec>::electra_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only() + .run(); + } + #[test] fn attester_slashing() { SszStaticHandler::, MinimalEthSpec>::pre_electra() From f85a1243623f27f228525277f8fb4d9f04276a52 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Thu, 20 Jun 2024 15:36:43 +0200 Subject: [PATCH 165/227] Electra attestation changes from Lions review (#5971) * dedup/cleanup and remove unneeded hashset use * remove irrelevant TODOs --- .../src/attestation_verification.rs | 3 +-- .../src/naive_aggregation_pool.rs | 24 +++++-------------- .../tests/attestation_verification.rs | 1 - .../src/common/get_attesting_indices.rs | 11 ++++----- .../per_block_processing/signature_sets.rs | 1 - 5 files changed, 11 insertions(+), 29 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 6044cafd6e0..32c33f3e617 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -1397,8 +1397,7 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( attesting_indices_electra::get_indexed_attestation(&committees, att) .map(|attestation| (attestation, committees_per_slot)) .map_err(|e| { - let index = att.committee_index().unwrap_or(0); - if e == BlockOperationError::BeaconStateError(NoCommitteeFound(index)) { + if let BlockOperationError::BeaconStateError(NoCommitteeFound(index)) = e { Error::NoCommitteeForSlotAndIndex { slot: att.data.slot, index, diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index b16ced789ff..211aecfe63d 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -241,24 +241,12 @@ impl AggregateMap for AggregatedAttestationMap { fn insert(&mut self, a: AttestationRef) -> Result { let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_CORE_INSERT); - let aggregation_bit = match a { - AttestationRef::Base(att) => att - .aggregation_bits - .iter() - .enumerate() - .filter_map(|(i, bit)| if bit { Some(i) } else { None }) - .at_most_one() - .map_err(|iter| Error::MoreThanOneAggregationBitSet(iter.count()))? - .ok_or(Error::NoAggregationBitsSet)?, - AttestationRef::Electra(att) => att - .aggregation_bits - .iter() - .enumerate() - .filter_map(|(i, bit)| if bit { Some(i) } else { None }) - .at_most_one() - .map_err(|iter| Error::MoreThanOneAggregationBitSet(iter.count()))? - .ok_or(Error::NoAggregationBitsSet)?, - }; + let aggregation_bit = *a + .set_aggregation_bits() + .iter() + .at_most_one() + .map_err(|iter| Error::MoreThanOneAggregationBitSet(iter.count()))? + .ok_or(Error::NoAggregationBitsSet)?; let attestation_key = AttestationKey::from_attestation_ref(a)?; let attestation_key_root = attestation_key.tree_hash_root(); diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index 3e35e58296a..19efe10c6d0 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -233,7 +233,6 @@ fn get_non_aggregator( let state = &head.beacon_state; let current_slot = chain.slot().expect("should get slot"); - // TODO(electra) make fork-agnostic let committee = state .get_beacon_committee( current_slot, diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index 457a5261041..b131f7679a3 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -47,7 +47,6 @@ pub mod attesting_indices_electra { use std::collections::HashSet; use crate::per_block_processing::errors::{AttestationInvalid as Invalid, BlockOperationError}; - use itertools::Itertools; use safe_arith::SafeArith; use types::*; @@ -96,7 +95,7 @@ pub mod attesting_indices_electra { aggregation_bits: &BitList, committee_bits: &BitVector, ) -> Result, BeaconStateError> { - let mut output: HashSet = HashSet::new(); + let mut attesting_indices = vec![]; let committee_indices = get_committee_indices::(committee_bits); @@ -128,8 +127,7 @@ pub mod attesting_indices_electra { }) .collect::>(); - output.extend(committee_attesters); - + attesting_indices.extend(committee_attesters); committee_offset.safe_add_assign(beacon_committee.committee.len())?; } @@ -138,10 +136,9 @@ pub mod attesting_indices_electra { return Err(BeaconStateError::InvalidBitfield); } - let mut indices = output.into_iter().collect_vec(); - indices.sort_unstable(); + attesting_indices.sort_unstable(); - Ok(indices) + Ok(attesting_indices) } pub fn get_committee_indices( diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index 8cf39abf437..2e00ee03418 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -326,7 +326,6 @@ where genesis_validators_root, ); - // TODO(electra), signing root isnt unique in the case of electra let message = indexed_attestation.data().signing_root(domain); Ok(SignatureSet::multiple_pubkeys(signature, pubkeys, message)) From 09f48c5527733b39672c4e854f5f3fd05683df17 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Thu, 20 Jun 2024 12:38:12 -0500 Subject: [PATCH 166/227] Fix Compilation Break --- validator_client/src/attestation_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 1a6504dd494..db32dbcf4a5 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -14,7 +14,7 @@ use std::ops::Deref; use std::sync::Arc; use tokio::time::{sleep, sleep_until, Duration, Instant}; use tree_hash::TreeHash; -use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot}; +use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, ForkName, Slot}; /// Builds an `AttestationService`. pub struct AttestationServiceBuilder { From fadd7aa86bc776191f319b3ad7a17af113b375ce Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Tue, 18 Jun 2024 13:42:23 -0500 Subject: [PATCH 167/227] Rename Constants, Structures, and Fields --- beacon_node/beacon_chain/src/eth1_chain.rs | 4 +- beacon_node/execution_layer/src/engine_api.rs | 20 +++--- .../src/engine_api/json_structures.rs | 30 ++++---- beacon_node/execution_layer/src/lib.rs | 8 +-- .../test_utils/execution_block_generator.rs | 2 +- .../src/test_utils/handle_rpc.rs | 4 +- beacon_node/store/src/partial_beacon_state.rs | 6 +- .../process_operations.rs | 14 ++-- .../state_processing/src/upgrade/electra.rs | 2 +- consensus/types/presets/gnosis/electra.yaml | 4 +- consensus/types/presets/mainnet/electra.yaml | 4 +- consensus/types/presets/minimal/electra.yaml | 4 +- consensus/types/src/beacon_block_body.rs | 2 +- consensus/types/src/beacon_state.rs | 2 +- consensus/types/src/chain_spec.rs | 6 +- consensus/types/src/config_and_preset.rs | 2 +- ...{deposit_receipt.rs => deposit_request.rs} | 4 +- consensus/types/src/eth_spec.rs | 28 ++++---- consensus/types/src/execution_payload.rs | 11 ++- .../types/src/execution_payload_header.rs | 6 +- consensus/types/src/lib.rs | 8 +-- consensus/types/src/payload.rs | 70 +++++++------------ consensus/types/src/preset.rs | 9 +-- ...rawal_request.rs => withdrawal_request.rs} | 4 +- testing/ef_tests/src/cases/operations.rs | 12 ++-- testing/ef_tests/src/type_name.rs | 4 +- testing/ef_tests/tests/tests.rs | 34 +++++---- 27 files changed, 141 insertions(+), 163 deletions(-) rename consensus/types/src/{deposit_receipt.rs => deposit_request.rs} (90%) rename consensus/types/src/{execution_layer_withdrawal_request.rs => withdrawal_request.rs} (84%) diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index 6e80abffc71..b4005f22fd1 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -548,8 +548,8 @@ impl Eth1ChainBackend for CachingEth1Backend { // [New in Electra:EIP6110] let deposit_index_limit = - if let Ok(deposit_receipts_start_index) = state.deposit_receipts_start_index() { - std::cmp::min(deposit_count, deposit_receipts_start_index) + if let Ok(deposit_requests_start_index) = state.deposit_requests_start_index() { + std::cmp::min(deposit_count, deposit_requests_start_index) } else { deposit_count }; diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 6bfd9997f43..e0bfb7d49eb 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -20,7 +20,7 @@ use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use strum::IntoStaticStr; use superstruct::superstruct; -use types::execution_payload::{DepositReceipts, WithdrawalRequests}; +use types::execution_payload::{DepositRequests, WithdrawalRequests}; pub use types::{ Address, BeaconBlockRef, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, @@ -67,7 +67,7 @@ pub enum Error { TransitionConfigurationMismatch, SszError(ssz_types::Error), DeserializeWithdrawals(ssz_types::Error), - DeserializeDepositReceipts(ssz_types::Error), + DeserializeDepositRequests(ssz_types::Error), DeserializeWithdrawalRequests(ssz_types::Error), BuilderApi(builder_client::Error), IncorrectStateVariant, @@ -204,7 +204,7 @@ pub struct ExecutionBlockWithTransactions { #[serde(with = "serde_utils::u64_hex_be")] pub excess_blob_gas: u64, #[superstruct(only(Electra))] - pub deposit_receipts: Vec, + pub deposit_requests: Vec, #[superstruct(only(Electra))] pub withdrawal_requests: Vec, } @@ -314,8 +314,8 @@ impl TryFrom> for ExecutionBlockWithTransactions .collect(), blob_gas_used: block.blob_gas_used, excess_blob_gas: block.excess_blob_gas, - deposit_receipts: block - .deposit_receipts + deposit_requests: block + .deposit_requests .into_iter() .map(|deposit| deposit.into()) .collect(), @@ -546,7 +546,7 @@ impl GetPayloadResponse { pub struct ExecutionPayloadBodyV1 { pub transactions: Transactions, pub withdrawals: Option>, - pub deposit_receipts: Option>, + pub deposit_requests: Option>, pub withdrawal_requests: Option>, } @@ -635,13 +635,13 @@ impl ExecutionPayloadBodyV1 { } } ExecutionPayloadHeader::Electra(header) => { - let (Some(withdrawals), Some(deposit_receipts), Some(withdrawal_requests)) = ( + let (Some(withdrawals), Some(deposit_requests), Some(withdrawal_requests)) = ( self.withdrawals, - self.deposit_receipts, + self.deposit_requests, self.withdrawal_requests, ) else { return Err(format!( - "block {} is post-electra but payload body doesn't have withdrawals/deposit_receipts/withdrawal_requests \ + "block {} is post-electra but payload body doesn't have withdrawals/deposit_requests/withdrawal_requests \ Check that ELs are returning receipts and withdrawal_requests in getPayloadBody requests", header.block_hash )); @@ -664,7 +664,7 @@ impl ExecutionPayloadBodyV1 { withdrawals, blob_gas_used: header.blob_gas_used, excess_blob_gas: header.excess_blob_gas, - deposit_receipts, + deposit_requests, withdrawal_requests, })) } diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index c41ee30e3d9..05754cfdb51 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -4,10 +4,7 @@ use strum::EnumString; use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::BlobsList; -use types::{ - DepositReceipt, ExecutionLayerWithdrawalRequest, FixedVector, PublicKeyBytes, Signature, - Unsigned, -}; +use types::{DepositRequest, FixedVector, PublicKeyBytes, Signature, Unsigned, WithdrawalRequest}; #[derive(Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -105,8 +102,7 @@ pub struct JsonExecutionPayload { #[serde(with = "serde_utils::u64_hex_be")] pub excess_blob_gas: u64, #[superstruct(only(V4))] - // TODO(electra): Field name should be changed post devnet-0. See https://github.com/ethereum/execution-apis/pull/544 - pub deposit_requests: VariableList, + pub deposit_requests: VariableList, #[superstruct(only(V4))] pub withdrawal_requests: VariableList, @@ -213,7 +209,7 @@ impl From> for JsonExecutionPayloadV4 blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, deposit_requests: payload - .deposit_receipts + .deposit_requests .into_iter() .map(Into::into) .collect::>() @@ -340,7 +336,7 @@ impl From> for ExecutionPayloadElectra .into(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, - deposit_receipts: payload + deposit_requests: payload .deposit_requests .into_iter() .map(Into::into) @@ -725,7 +721,7 @@ pub struct JsonExecutionPayloadBodyV1 { #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, pub withdrawals: Option>, - pub deposit_receipts: Option>, + pub deposit_requests: Option>, pub withdrawal_requests: Option>, } @@ -742,8 +738,8 @@ impl From> for ExecutionPayloadBodyV1< .collect::>(), ) }), - deposit_receipts: value.deposit_receipts.map(|json_receipts| { - DepositReceipts::::from( + deposit_requests: value.deposit_requests.map(|json_receipts| { + DepositRequests::::from( json_receipts .into_iter() .map(Into::into) @@ -849,8 +845,8 @@ pub struct JsonDepositRequest { pub index: u64, } -impl From for JsonDepositRequest { - fn from(deposit: DepositReceipt) -> Self { +impl From for JsonDepositRequest { + fn from(deposit: DepositRequest) -> Self { Self { pubkey: deposit.pubkey, withdrawal_credentials: deposit.withdrawal_credentials, @@ -861,7 +857,7 @@ impl From for JsonDepositRequest { } } -impl From for DepositReceipt { +impl From for DepositRequest { fn from(json_deposit: JsonDepositRequest) -> Self { Self { pubkey: json_deposit.pubkey, @@ -882,8 +878,8 @@ pub struct JsonWithdrawalRequest { pub amount: u64, } -impl From for JsonWithdrawalRequest { - fn from(withdrawal_request: ExecutionLayerWithdrawalRequest) -> Self { +impl From for JsonWithdrawalRequest { + fn from(withdrawal_request: WithdrawalRequest) -> Self { Self { source_address: withdrawal_request.source_address, validator_public_key: withdrawal_request.validator_pubkey, @@ -892,7 +888,7 @@ impl From for JsonWithdrawalRequest { } } -impl From for ExecutionLayerWithdrawalRequest { +impl From for WithdrawalRequest { fn from(json_withdrawal_request: JsonWithdrawalRequest) -> Self { Self { source_address: json_withdrawal_request.source_address, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 6ac4b6eb9a5..eaa739d7a5d 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1994,14 +1994,14 @@ impl ExecutionLayer { .collect(), ) .map_err(ApiError::DeserializeWithdrawals)?; - let deposit_receipts = VariableList::new( + let deposit_requests = VariableList::new( electra_block - .deposit_receipts + .deposit_requests .into_iter() .map(Into::into) .collect(), ) - .map_err(ApiError::DeserializeDepositReceipts)?; + .map_err(ApiError::DeserializeDepositRequests)?; let withdrawal_requests = VariableList::new( electra_block .withdrawal_requests @@ -2028,7 +2028,7 @@ impl ExecutionLayer { withdrawals, blob_gas_used: electra_block.blob_gas_used, excess_blob_gas: electra_block.excess_blob_gas, - deposit_receipts, + deposit_requests, withdrawal_requests, }) } diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index e80c6b23705..8619e24a238 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -659,7 +659,7 @@ impl ExecutionBlockGenerator { withdrawals: pa.withdrawals.clone().into(), blob_gas_used: 0, excess_blob_gas: 0, - deposit_receipts: vec![].into(), + deposit_requests: vec![].into(), withdrawal_requests: vec![].into(), }), _ => unreachable!(), diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 9f1f244ef9a..0dc7a7759c5 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -589,8 +589,8 @@ pub async fn handle_rpc( .withdrawals() .ok() .map(|withdrawals| VariableList::from(withdrawals.clone())), - deposit_receipts: block.deposit_receipts().ok().map( - |deposit_receipts| VariableList::from(deposit_receipts.clone()), + deposit_requests: block.deposit_requests().ok().map( + |deposit_requests| VariableList::from(deposit_requests.clone()), ), withdrawal_requests: block.withdrawal_requests().ok().map( |withdrawal_requests| { diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index 5e6054bc06b..8f40b4b9241 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -121,7 +121,7 @@ where // Electra #[superstruct(only(Electra))] - pub deposit_receipts_start_index: u64, + pub deposit_requests_start_index: u64, #[superstruct(only(Electra))] pub deposit_balance_to_consume: u64, #[superstruct(only(Electra))] @@ -284,7 +284,7 @@ impl PartialBeaconState { latest_execution_payload_header, next_withdrawal_index, next_withdrawal_validator_index, - deposit_receipts_start_index, + deposit_requests_start_index, deposit_balance_to_consume, exit_balance_to_consume, earliest_exit_epoch, @@ -557,7 +557,7 @@ impl TryInto> for PartialBeaconState { latest_execution_payload_header, next_withdrawal_index, next_withdrawal_validator_index, - deposit_receipts_start_index, + deposit_requests_start_index, deposit_balance_to_consume, exit_balance_to_consume, earliest_exit_epoch, diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 02cf1d942cc..06450922fd7 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -44,7 +44,7 @@ pub fn process_operations>( if let Some(requests) = requests { process_execution_layer_withdrawal_requests(state, &requests, spec)?; } - let receipts = block_body.execution_payload()?.deposit_receipts()?; + let receipts = block_body.execution_payload()?.deposit_requests()?; if let Some(receipts) = receipts { process_deposit_receipts(state, &receipts, spec)?; } @@ -372,9 +372,9 @@ pub fn process_deposits( // [Modified in Electra:EIP6110] // Disable former deposit mechanism once all prior deposits are processed // - // If `deposit_receipts_start_index` does not exist as a field on `state`, electra is disabled + // If `deposit_requests_start_index` does not exist as a field on `state`, electra is disabled // which means we always want to use the old check, so this field defaults to `u64::MAX`. - let eth1_deposit_index_limit = state.deposit_receipts_start_index().unwrap_or(u64::MAX); + let eth1_deposit_index_limit = state.deposit_requests_start_index().unwrap_or(u64::MAX); if state.eth1_deposit_index() < eth1_deposit_index_limit { let expected_deposit_len = std::cmp::min( @@ -530,7 +530,7 @@ pub fn apply_deposit( pub fn process_execution_layer_withdrawal_requests( state: &mut BeaconState, - requests: &[ExecutionLayerWithdrawalRequest], + requests: &[WithdrawalRequest], spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { for request in requests { @@ -627,13 +627,13 @@ pub fn process_execution_layer_withdrawal_requests( pub fn process_deposit_receipts( state: &mut BeaconState, - receipts: &[DepositReceipt], + receipts: &[DepositRequest], spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { for receipt in receipts { // Set deposit receipt start index - if state.deposit_receipts_start_index()? == spec.unset_deposit_receipts_start_index { - *state.deposit_receipts_start_index_mut()? = receipt.index + if state.deposit_requests_start_index()? == spec.unset_deposit_requests_start_index { + *state.deposit_requests_start_index_mut()? = receipt.index } let deposit_data = DepositData { pubkey: receipt.pubkey, diff --git a/consensus/state_processing/src/upgrade/electra.rs b/consensus/state_processing/src/upgrade/electra.rs index 8d83e738282..1e532d9f107 100644 --- a/consensus/state_processing/src/upgrade/electra.rs +++ b/consensus/state_processing/src/upgrade/electra.rs @@ -131,7 +131,7 @@ pub fn upgrade_state_to_electra( next_withdrawal_validator_index: pre.next_withdrawal_validator_index, historical_summaries: pre.historical_summaries.clone(), // Electra - deposit_receipts_start_index: spec.unset_deposit_receipts_start_index, + deposit_requests_start_index: spec.unset_deposit_requests_start_index, deposit_balance_to_consume: 0, exit_balance_to_consume: 0, earliest_exit_epoch, diff --git a/consensus/types/presets/gnosis/electra.yaml b/consensus/types/presets/gnosis/electra.yaml index 72c626ded2f..660ed9b64cf 100644 --- a/consensus/types/presets/gnosis/electra.yaml +++ b/consensus/types/presets/gnosis/electra.yaml @@ -30,12 +30,12 @@ MAX_ATTESTER_SLASHINGS_ELECTRA: 1 # `uint64(2**3)` (= 8) MAX_ATTESTATIONS_ELECTRA: 8 # `uint64(2**0)` (= 1) -MAX_CONSOLIDATIONS: 1 +MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1 # Execution # --------------------------------------------------------------- # 2**13 (= 8192) receipts -MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192 +MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: 8192 # 2**4 (= 16) withdrawal requests MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16 diff --git a/consensus/types/presets/mainnet/electra.yaml b/consensus/types/presets/mainnet/electra.yaml index 72c626ded2f..660ed9b64cf 100644 --- a/consensus/types/presets/mainnet/electra.yaml +++ b/consensus/types/presets/mainnet/electra.yaml @@ -30,12 +30,12 @@ MAX_ATTESTER_SLASHINGS_ELECTRA: 1 # `uint64(2**3)` (= 8) MAX_ATTESTATIONS_ELECTRA: 8 # `uint64(2**0)` (= 1) -MAX_CONSOLIDATIONS: 1 +MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1 # Execution # --------------------------------------------------------------- # 2**13 (= 8192) receipts -MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192 +MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: 8192 # 2**4 (= 16) withdrawal requests MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16 diff --git a/consensus/types/presets/minimal/electra.yaml b/consensus/types/presets/minimal/electra.yaml index 11aa5e1f50e..ef1ce494d8e 100644 --- a/consensus/types/presets/minimal/electra.yaml +++ b/consensus/types/presets/minimal/electra.yaml @@ -30,12 +30,12 @@ MAX_ATTESTER_SLASHINGS_ELECTRA: 1 # `uint64(2**3)` (= 8) MAX_ATTESTATIONS_ELECTRA: 8 # `uint64(2**0)` (= 1) -MAX_CONSOLIDATIONS: 1 +MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1 # Execution # --------------------------------------------------------------- # [customized] -MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4 +MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: 4 # [customized] 2**1 (= 2) withdrawal requests MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 2 diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index 9cf66184f8d..199f6c1c8b1 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -115,7 +115,7 @@ pub struct BeaconBlockBody = FullPay #[superstruct(only(Deneb, Electra))] pub blob_kzg_commitments: KzgCommitments, #[superstruct(only(Electra))] - pub consolidations: VariableList, + pub consolidations: VariableList, #[superstruct(only(Base, Altair))] #[metastruct(exclude_from(fields))] #[ssz(skip_serializing, skip_deserializing)] diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index b0da5b7eed3..295aaade1cf 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -484,7 +484,7 @@ where #[superstruct(only(Electra), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] #[serde(with = "serde_utils::quoted_u64")] - pub deposit_receipts_start_index: u64, + pub deposit_requests_start_index: u64, #[superstruct(only(Electra), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index a8c087d6d2f..69370c9b986 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -180,7 +180,7 @@ pub struct ChainSpec { pub electra_fork_version: [u8; 4], /// The Electra fork epoch is optional, with `None` representing "Electra never happens". pub electra_fork_epoch: Option, - pub unset_deposit_receipts_start_index: u64, + pub unset_deposit_requests_start_index: u64, pub full_exit_request_amount: u64, pub min_activation_balance: u64, pub max_effective_balance_electra: u64, @@ -747,7 +747,7 @@ impl ChainSpec { */ electra_fork_version: [0x05, 00, 00, 00], electra_fork_epoch: None, - unset_deposit_receipts_start_index: u64::MAX, + unset_deposit_requests_start_index: u64::MAX, full_exit_request_amount: 0, min_activation_balance: option_wrapper(|| { u64::checked_pow(2, 5)?.checked_mul(u64::checked_pow(10, 9)?) @@ -1057,7 +1057,7 @@ impl ChainSpec { */ electra_fork_version: [0x05, 0x00, 0x00, 0x64], electra_fork_epoch: None, - unset_deposit_receipts_start_index: u64::MAX, + unset_deposit_requests_start_index: u64::MAX, full_exit_request_amount: 0, min_activation_balance: option_wrapper(|| { u64::checked_pow(2, 5)?.checked_mul(u64::checked_pow(10, 9)?) diff --git a/consensus/types/src/config_and_preset.rs b/consensus/types/src/config_and_preset.rs index 6fc6e0642ea..110392d4b77 100644 --- a/consensus/types/src/config_and_preset.rs +++ b/consensus/types/src/config_and_preset.rs @@ -124,7 +124,7 @@ pub fn get_extra_fields(spec: &ChainSpec) -> HashMap { "versioned_hash_version_kzg".to_uppercase() => deneb::VERSIONED_HASH_VERSION_KZG.to_string().into(), // Electra "compounding_withdrawal_prefix".to_uppercase() => u8_hex(spec.compounding_withdrawal_prefix_byte), - "unset_deposit_receipts_start_index".to_uppercase() => spec.unset_deposit_receipts_start_index.to_string().into(), + "unset_deposit_requests_start_index".to_uppercase() => spec.unset_deposit_requests_start_index.to_string().into(), "full_exit_request_amount".to_uppercase() => spec.full_exit_request_amount.to_string().into(), "domain_consolidation".to_uppercase()=> u32_hex(spec.domain_consolidation), } diff --git a/consensus/types/src/deposit_receipt.rs b/consensus/types/src/deposit_request.rs similarity index 90% rename from consensus/types/src/deposit_receipt.rs rename to consensus/types/src/deposit_request.rs index 6a08f717f3d..f6ddf8b63a8 100644 --- a/consensus/types/src/deposit_receipt.rs +++ b/consensus/types/src/deposit_request.rs @@ -19,7 +19,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] -pub struct DepositReceipt { +pub struct DepositRequest { pub pubkey: PublicKeyBytes, pub withdrawal_credentials: Hash256, #[serde(with = "serde_utils::quoted_u64")] @@ -33,5 +33,5 @@ pub struct DepositReceipt { mod tests { use super::*; - ssz_and_tree_hash_tests!(DepositReceipt); + ssz_and_tree_hash_tests!(DepositRequest); } diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 1c379f5de42..262fa69696f 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -143,8 +143,8 @@ pub trait EthSpec: type PendingBalanceDepositsLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; type PendingPartialWithdrawalsLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; type PendingConsolidationsLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; - type MaxConsolidations: Unsigned + Clone + Sync + Send + Debug + PartialEq; - type MaxDepositReceiptsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxConsolidationRequestsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxDepositRequestsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxAttesterSlashingsElectra: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxAttestationsElectra: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxWithdrawalRequestsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; @@ -325,14 +325,14 @@ pub trait EthSpec: Self::PendingConsolidationsLimit::to_usize() } - /// Returns the `MAX_CONSOLIDATIONS` constant for this specification. - fn max_consolidations() -> usize { - Self::MaxConsolidations::to_usize() + /// Returns the `MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD` constant for this specification. + fn max_consolidation_requests_per_payload() -> usize { + Self::MaxConsolidationRequestsPerPayload::to_usize() } - /// Returns the `MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD` constant for this specification. - fn max_deposit_receipts_per_payload() -> usize { - Self::MaxDepositReceiptsPerPayload::to_usize() + /// Returns the `MAX_DEPOSIT_REQUESTS_PER_PAYLOAD` constant for this specification. + fn max_deposit_requests_per_payload() -> usize { + Self::MaxDepositRequestsPerPayload::to_usize() } /// Returns the `MAX_ATTESTER_SLASHINGS_ELECTRA` constant for this specification. @@ -404,8 +404,8 @@ impl EthSpec for MainnetEthSpec { type PendingBalanceDepositsLimit = U134217728; type PendingPartialWithdrawalsLimit = U134217728; type PendingConsolidationsLimit = U262144; - type MaxConsolidations = U1; - type MaxDepositReceiptsPerPayload = U8192; + type MaxConsolidationRequestsPerPayload = U1; + type MaxDepositRequestsPerPayload = U8192; type MaxAttesterSlashingsElectra = U1; type MaxAttestationsElectra = U8; type MaxWithdrawalRequestsPerPayload = U16; @@ -442,7 +442,7 @@ impl EthSpec for MinimalEthSpec { type KzgCommitmentInclusionProofDepth = U9; type PendingPartialWithdrawalsLimit = U64; type PendingConsolidationsLimit = U64; - type MaxDepositReceiptsPerPayload = U4; + type MaxDepositRequestsPerPayload = U4; type MaxWithdrawalRequestsPerPayload = U2; params_from_eth_spec!(MainnetEthSpec { @@ -468,7 +468,7 @@ impl EthSpec for MinimalEthSpec { MaxBlobsPerBlock, BytesPerFieldElement, PendingBalanceDepositsLimit, - MaxConsolidations, + MaxConsolidationRequestsPerPayload, MaxAttesterSlashingsElectra, MaxAttestationsElectra }); @@ -527,8 +527,8 @@ impl EthSpec for GnosisEthSpec { type PendingBalanceDepositsLimit = U134217728; type PendingPartialWithdrawalsLimit = U134217728; type PendingConsolidationsLimit = U262144; - type MaxConsolidations = U1; - type MaxDepositReceiptsPerPayload = U8192; + type MaxConsolidationRequestsPerPayload = U1; + type MaxDepositRequestsPerPayload = U8192; type MaxAttesterSlashingsElectra = U1; type MaxAttestationsElectra = U8; type MaxWithdrawalRequestsPerPayload = U16; diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index cd6667b3751..aa6ad8bc95b 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -13,10 +13,10 @@ pub type Transactions = VariableList< >; pub type Withdrawals = VariableList::MaxWithdrawalsPerPayload>; -pub type DepositReceipts = - VariableList::MaxDepositReceiptsPerPayload>; +pub type DepositRequests = + VariableList::MaxDepositRequestsPerPayload>; pub type WithdrawalRequests = - VariableList::MaxWithdrawalRequestsPerPayload>; + VariableList::MaxWithdrawalRequestsPerPayload>; #[superstruct( variants(Bellatrix, Capella, Deneb, Electra), @@ -94,10 +94,9 @@ pub struct ExecutionPayload { #[serde(with = "serde_utils::quoted_u64")] pub excess_blob_gas: u64, #[superstruct(only(Electra))] - pub deposit_receipts: VariableList, + pub deposit_requests: VariableList, #[superstruct(only(Electra))] - pub withdrawal_requests: - VariableList, + pub withdrawal_requests: VariableList, } impl<'a, E: EthSpec> ExecutionPayloadRef<'a, E> { diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index a3df69652c2..fe94c8024e7 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -86,7 +86,7 @@ pub struct ExecutionPayloadHeader { #[serde(with = "serde_utils::quoted_u64")] pub excess_blob_gas: u64, #[superstruct(only(Electra), partial_getter(copy))] - pub deposit_receipts_root: Hash256, + pub deposit_requests_root: Hash256, #[superstruct(only(Electra), partial_getter(copy))] pub withdrawal_requests_root: Hash256, } @@ -206,7 +206,7 @@ impl ExecutionPayloadHeaderDeneb { withdrawals_root: self.withdrawals_root, blob_gas_used: self.blob_gas_used, excess_blob_gas: self.excess_blob_gas, - deposit_receipts_root: Hash256::zero(), + deposit_requests_root: Hash256::zero(), withdrawal_requests_root: Hash256::zero(), } } @@ -299,7 +299,7 @@ impl<'a, E: EthSpec> From<&'a ExecutionPayloadElectra> for ExecutionPayloadHe withdrawals_root: payload.withdrawals.tree_hash_root(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, - deposit_receipts_root: payload.deposit_receipts.tree_hash_root(), + deposit_requests_root: payload.deposit_requests.tree_hash_root(), withdrawal_requests_root: payload.withdrawal_requests.tree_hash_root(), } } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 2a73ff0f37a..66898b1385e 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -33,13 +33,12 @@ pub mod contribution_and_proof; pub mod deposit; pub mod deposit_data; pub mod deposit_message; -pub mod deposit_receipt; +pub mod deposit_request; pub mod deposit_tree_snapshot; pub mod enr_fork_id; pub mod eth1_data; pub mod eth_spec; pub mod execution_block_hash; -pub mod execution_layer_withdrawal_request; pub mod execution_payload; pub mod execution_payload_header; pub mod fork; @@ -77,6 +76,7 @@ pub mod validator; pub mod validator_subscription; pub mod voluntary_exit; pub mod withdrawal_credentials; +pub mod withdrawal_request; #[macro_use] pub mod slot_epoch_macros; pub mod activation_queue; @@ -150,7 +150,7 @@ pub use crate::contribution_and_proof::ContributionAndProof; pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH}; pub use crate::deposit_data::DepositData; pub use crate::deposit_message::DepositMessage; -pub use crate::deposit_receipt::DepositReceipt; +pub use crate::deposit_request::DepositRequest; pub use crate::deposit_tree_snapshot::{DepositTreeSnapshot, FinalizedExecutionBlock}; pub use crate::enr_fork_id::EnrForkId; pub use crate::epoch_cache::{EpochCache, EpochCacheError, EpochCacheKey}; @@ -158,7 +158,6 @@ pub use crate::eth1_data::Eth1Data; pub use crate::eth_spec::EthSpecId; pub use crate::execution_block_hash::ExecutionBlockHash; pub use crate::execution_block_header::ExecutionBlockHeader; -pub use crate::execution_layer_withdrawal_request::ExecutionLayerWithdrawalRequest; pub use crate::execution_payload::{ ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadRef, Transaction, Transactions, Withdrawals, @@ -251,6 +250,7 @@ pub use crate::validator_subscription::ValidatorSubscription; pub use crate::voluntary_exit::VoluntaryExit; pub use crate::withdrawal::Withdrawal; pub use crate::withdrawal_credentials::WithdrawalCredentials; +pub use crate::withdrawal_request::WithdrawalRequest; pub type CommitteeIndex = u64; pub type Hash256 = H256; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 644d401ec7e..4f1a8574623 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -41,13 +41,10 @@ pub trait ExecPayload: Debug + Clone + PartialEq + Hash + TreeHash + fn blob_gas_used(&self) -> Result; fn withdrawal_requests( &self, - ) -> Result< - Option>, - Error, - >; - fn deposit_receipts( + ) -> Result>, Error>; + fn deposit_requests( &self, - ) -> Result>, Error>; + ) -> Result>, Error>; /// Is this a default payload with 0x0 roots for transactions and withdrawals? fn is_default_with_zero_roots(&self) -> bool; @@ -289,10 +286,8 @@ impl ExecPayload for FullPayload { fn withdrawal_requests( &self, - ) -> Result< - Option>, - Error, - > { + ) -> Result>, Error> + { match self { FullPayload::Bellatrix(_) | FullPayload::Capella(_) | FullPayload::Deneb(_) => { Err(Error::IncorrectStateVariant) @@ -303,15 +298,15 @@ impl ExecPayload for FullPayload { } } - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error> { + ) -> Result>, Error> { match self { FullPayload::Bellatrix(_) | FullPayload::Capella(_) | FullPayload::Deneb(_) => { Err(Error::IncorrectStateVariant) } FullPayload::Electra(inner) => { - Ok(Some(inner.execution_payload.deposit_receipts.clone())) + Ok(Some(inner.execution_payload.deposit_requests.clone())) } } } @@ -450,10 +445,8 @@ impl<'b, E: EthSpec> ExecPayload for FullPayloadRef<'b, E> { fn withdrawal_requests( &self, - ) -> Result< - Option>, - Error, - > { + ) -> Result>, Error> + { match self { FullPayloadRef::Bellatrix(_) | FullPayloadRef::Capella(_) @@ -464,15 +457,15 @@ impl<'b, E: EthSpec> ExecPayload for FullPayloadRef<'b, E> { } } - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error> { + ) -> Result>, Error> { match self { FullPayloadRef::Bellatrix(_) | FullPayloadRef::Capella(_) | FullPayloadRef::Deneb(_) => Err(Error::IncorrectStateVariant), FullPayloadRef::Electra(inner) => { - Ok(Some(inner.execution_payload.deposit_receipts.clone())) + Ok(Some(inner.execution_payload.deposit_requests.clone())) } } } @@ -659,16 +652,14 @@ impl ExecPayload for BlindedPayload { fn withdrawal_requests( &self, - ) -> Result< - Option>, - Error, - > { + ) -> Result>, Error> + { Ok(None) } - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error> { + ) -> Result>, Error> { Ok(None) } @@ -775,16 +766,14 @@ impl<'b, E: EthSpec> ExecPayload for BlindedPayloadRef<'b, E> { fn withdrawal_requests( &self, - ) -> Result< - Option>, - Error, - > { + ) -> Result>, Error> + { Ok(None) } - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error> { + ) -> Result>, Error> { Ok(None) } @@ -883,16 +872,16 @@ macro_rules! impl_exec_payload_common { fn withdrawal_requests( &self, ) -> Result< - Option>, + Option>, Error, > { let i = $i; i(self) } - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error> { + ) -> Result>, Error> { let j = $j; j(self) } @@ -1035,12 +1024,7 @@ macro_rules! impl_exec_payload_for_fork { let c: for<'a> fn( &'a $wrapper_type_full, ) -> Result< - Option< - VariableList< - ExecutionLayerWithdrawalRequest, - E::MaxWithdrawalRequestsPerPayload, - >, - >, + Option>, Error, > = |payload: &$wrapper_type_full| { let wrapper_ref_type = FullPayloadRef::$fork_variant(&payload); @@ -1052,11 +1036,11 @@ macro_rules! impl_exec_payload_for_fork { let c: for<'a> fn( &'a $wrapper_type_full, ) -> Result< - Option>, + Option>, Error, > = |payload: &$wrapper_type_full| { let wrapper_ref_type = FullPayloadRef::$fork_variant(&payload); - wrapper_ref_type.deposit_receipts() + wrapper_ref_type.deposit_requests() }; c } diff --git a/consensus/types/src/preset.rs b/consensus/types/src/preset.rs index f4008d62e1d..a1eadaff253 100644 --- a/consensus/types/src/preset.rs +++ b/consensus/types/src/preset.rs @@ -246,9 +246,9 @@ pub struct ElectraPreset { #[serde(with = "serde_utils::quoted_u64")] pub pending_consolidations_limit: u64, #[serde(with = "serde_utils::quoted_u64")] - pub max_consolidations: u64, + pub max_consolidation_requests_per_payload: u64, #[serde(with = "serde_utils::quoted_u64")] - pub max_deposit_receipts_per_payload: u64, + pub max_deposit_requests_per_payload: u64, #[serde(with = "serde_utils::quoted_u64")] pub max_attester_slashings_electra: u64, #[serde(with = "serde_utils::quoted_u64")] @@ -269,8 +269,9 @@ impl ElectraPreset { pending_balance_deposits_limit: E::pending_balance_deposits_limit() as u64, pending_partial_withdrawals_limit: E::pending_partial_withdrawals_limit() as u64, pending_consolidations_limit: E::pending_consolidations_limit() as u64, - max_consolidations: E::max_consolidations() as u64, - max_deposit_receipts_per_payload: E::max_deposit_receipts_per_payload() as u64, + max_consolidation_requests_per_payload: E::max_consolidation_requests_per_payload() + as u64, + max_deposit_requests_per_payload: E::max_deposit_requests_per_payload() as u64, max_attester_slashings_electra: E::max_attester_slashings_electra() as u64, max_attestations_electra: E::max_attestations_electra() as u64, max_withdrawal_requests_per_payload: E::max_withdrawal_requests_per_payload() as u64, diff --git a/consensus/types/src/execution_layer_withdrawal_request.rs b/consensus/types/src/withdrawal_request.rs similarity index 84% rename from consensus/types/src/execution_layer_withdrawal_request.rs rename to consensus/types/src/withdrawal_request.rs index b1d814c2834..2f69e9b1504 100644 --- a/consensus/types/src/execution_layer_withdrawal_request.rs +++ b/consensus/types/src/withdrawal_request.rs @@ -19,7 +19,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] -pub struct ExecutionLayerWithdrawalRequest { +pub struct WithdrawalRequest { pub source_address: Address, pub validator_pubkey: PublicKeyBytes, #[serde(with = "serde_utils::quoted_u64")] @@ -30,5 +30,5 @@ pub struct ExecutionLayerWithdrawalRequest { mod tests { use super::*; - ssz_and_tree_hash_tests!(ExecutionLayerWithdrawalRequest); + ssz_and_tree_hash_tests!(WithdrawalRequest); } diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index bb82674c88a..cf684c38e36 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -25,9 +25,9 @@ use std::fmt::Debug; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockBodyBellatrix, BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconState, - BlindedPayload, Deposit, DepositReceipt, ExecutionLayerWithdrawalRequest, ExecutionPayload, - FullPayload, ProposerSlashing, SignedBlsToExecutionChange, SignedConsolidation, - SignedVoluntaryExit, SyncAggregate, + BlindedPayload, Deposit, DepositRequest, ExecutionPayload, FullPayload, ProposerSlashing, + SignedBlsToExecutionChange, SignedConsolidation, SignedVoluntaryExit, SyncAggregate, + WithdrawalRequest, }; #[derive(Debug, Clone, Default, Deserialize)] @@ -445,7 +445,7 @@ impl Operation for SignedBlsToExecutionChange { } } -impl Operation for ExecutionLayerWithdrawalRequest { +impl Operation for WithdrawalRequest { fn handler_name() -> String { "execution_layer_withdrawal_request".into() } @@ -468,9 +468,9 @@ impl Operation for ExecutionLayerWithdrawalRequest { } } -impl Operation for DepositReceipt { +impl Operation for DepositRequest { fn handler_name() -> String { - "deposit_receipt".into() + "deposit_request".into() } fn is_enabled_for_fork(fork_name: ForkName) -> bool { diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 6141fd902c6..19e76f3c480 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -63,9 +63,9 @@ type_name_generic!(ContributionAndProof); type_name!(Deposit); type_name!(DepositData); type_name!(DepositMessage); -type_name!(DepositReceipt); +type_name!(DepositRequest); type_name!(Eth1Data); -type_name!(ExecutionLayerWithdrawalRequest); +type_name!(WithdrawalRequest); type_name_generic!(ExecutionPayload); type_name_generic!(ExecutionPayloadBellatrix, "ExecutionPayload"); type_name_generic!(ExecutionPayloadCapella, "ExecutionPayload"); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 80b5cb458ab..a45125f1397 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -1,7 +1,7 @@ #![cfg(feature = "ef_tests")] use ef_tests::*; -use types::{ExecutionLayerWithdrawalRequest, MainnetEthSpec, MinimalEthSpec, *}; +use types::{MainnetEthSpec, MinimalEthSpec, WithdrawalRequest, *}; // Check that the hand-computed multiplications on EthSpec are correctly computed. // This test lives here because one is most likely to muck these up during a spec update. @@ -93,16 +93,16 @@ fn operations_withdrawals() { } #[test] -fn operations_execution_layer_withdrawal_reqeusts() { - OperationsHandler::::default().run(); - OperationsHandler::::default().run(); +fn operations_withdrawal_reqeusts() { + OperationsHandler::::default().run(); + OperationsHandler::::default().run(); } #[test] #[cfg(not(feature = "fake_crypto"))] -fn operations_deposit_receipts() { - OperationsHandler::::default().run(); - OperationsHandler::::default().run(); +fn operations_deposit_requests() { + OperationsHandler::::default().run(); + OperationsHandler::::default().run(); } #[test] @@ -242,9 +242,9 @@ mod ssz_static { use types::blob_sidecar::BlobIdentifier; use types::historical_summary::HistoricalSummary; use types::{ - AttesterSlashingBase, AttesterSlashingElectra, Consolidation, DepositReceipt, - ExecutionLayerWithdrawalRequest, LightClientBootstrapAltair, PendingBalanceDeposit, - PendingPartialWithdrawal, *, + AttesterSlashingBase, AttesterSlashingElectra, Consolidation, DepositRequest, + LightClientBootstrapAltair, PendingBalanceDeposit, PendingPartialWithdrawal, + WithdrawalRequest, *, }; ssz_static_test!(attestation_data, AttestationData); @@ -608,17 +608,15 @@ mod ssz_static { } #[test] - fn deposit_receipt() { - SszStaticHandler::::electra_and_later().run(); - SszStaticHandler::::electra_and_later().run(); + fn deposit_request() { + SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); } #[test] - fn execution_layer_withdrawal_request() { - SszStaticHandler::::electra_and_later() - .run(); - SszStaticHandler::::electra_and_later() - .run(); + fn withdrawal_request() { + SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); } #[test] From 37f1146f3844c2a682fad7c2e51f90d0b47d0464 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Thu, 20 Jun 2024 10:35:24 -0500 Subject: [PATCH 168/227] Second Round --- .../src/{consolidation.rs => consolidation_request.rs} | 6 +++--- consensus/types/src/lib.rs | 4 ++-- consensus/types/src/signed_consolidation.rs | 4 ++-- testing/ef_tests/src/type_name.rs | 2 +- testing/ef_tests/tests/tests.rs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) rename consensus/types/src/{consolidation.rs => consolidation_request.rs} (82%) diff --git a/consensus/types/src/consolidation.rs b/consensus/types/src/consolidation_request.rs similarity index 82% rename from consensus/types/src/consolidation.rs rename to consensus/types/src/consolidation_request.rs index 6cc4aa90f27..7f185a3827b 100644 --- a/consensus/types/src/consolidation.rs +++ b/consensus/types/src/consolidation_request.rs @@ -19,7 +19,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] -pub struct Consolidation { +pub struct ConsolidationRequest { #[serde(with = "serde_utils::quoted_u64")] pub source_index: u64, #[serde(with = "serde_utils::quoted_u64")] @@ -27,11 +27,11 @@ pub struct Consolidation { pub epoch: Epoch, } -impl SignedRoot for Consolidation {} +impl SignedRoot for ConsolidationRequest {} #[cfg(test)] mod tests { use super::*; - ssz_and_tree_hash_tests!(Consolidation); + ssz_and_tree_hash_tests!(ConsolidationRequest); } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 66898b1385e..cf3e209fad6 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -27,7 +27,7 @@ pub mod bls_to_execution_change; pub mod builder_bid; pub mod chain_spec; pub mod checkpoint; -pub mod consolidation; +pub mod consolidation_request; pub mod consts; pub mod contribution_and_proof; pub mod deposit; @@ -145,7 +145,7 @@ pub use crate::checkpoint::Checkpoint; pub use crate::config_and_preset::{ ConfigAndPreset, ConfigAndPresetCapella, ConfigAndPresetDeneb, ConfigAndPresetElectra, }; -pub use crate::consolidation::Consolidation; +pub use crate::consolidation_request::ConsolidationRequest; pub use crate::contribution_and_proof::ContributionAndProof; pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH}; pub use crate::deposit_data::DepositData; diff --git a/consensus/types/src/signed_consolidation.rs b/consensus/types/src/signed_consolidation.rs index f004ec23bd4..c6316995b37 100644 --- a/consensus/types/src/signed_consolidation.rs +++ b/consensus/types/src/signed_consolidation.rs @@ -1,5 +1,5 @@ use crate::test_utils::TestRandom; -use crate::{Consolidation, Signature}; +use crate::{ConsolidationRequest, Signature}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -20,7 +20,7 @@ use tree_hash_derive::TreeHash; TestRandom, )] pub struct SignedConsolidation { - pub message: Consolidation, + pub message: ConsolidationRequest, pub signature: Signature, } diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 19e76f3c480..319e5c49c89 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -58,7 +58,7 @@ type_name_generic!(BeaconState); type_name!(BlobIdentifier); type_name_generic!(BlobSidecar); type_name!(Checkpoint); -type_name!(Consolidation); +type_name!(ConsolidationRequest); type_name_generic!(ContributionAndProof); type_name!(Deposit); type_name!(DepositData); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index a45125f1397..99028e15458 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -242,7 +242,7 @@ mod ssz_static { use types::blob_sidecar::BlobIdentifier; use types::historical_summary::HistoricalSummary; use types::{ - AttesterSlashingBase, AttesterSlashingElectra, Consolidation, DepositRequest, + AttesterSlashingBase, AttesterSlashingElectra, ConsolidationRequest, DepositRequest, LightClientBootstrapAltair, PendingBalanceDeposit, PendingPartialWithdrawal, WithdrawalRequest, *, }; From bcecd80d18c059c5ce1ee57b2a90ee868862d5d5 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Thu, 20 Jun 2024 14:54:18 -0500 Subject: [PATCH 169/227] Third Round --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 - .../block_signature_verifier.rs | 22 --- .../process_operations.rs | 149 +----------------- .../per_block_processing/signature_sets.rs | 38 +---- consensus/types/src/beacon_block.rs | 2 - consensus/types/src/beacon_block_body.rs | 6 - consensus/types/src/consolidation_request.rs | 11 +- consensus/types/src/lib.rs | 2 - consensus/types/src/signed_beacon_block.rs | 2 - testing/ef_tests/src/cases/operations.rs | 29 +--- testing/ef_tests/src/type_name.rs | 1 - testing/ef_tests/tests/tests.rs | 10 +- 12 files changed, 15 insertions(+), 259 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 66e2964669e..e270cc8d898 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -5359,8 +5359,6 @@ impl BeaconChain { bls_to_execution_changes: bls_to_execution_changes.into(), blob_kzg_commitments: kzg_commitments .ok_or(BlockProductionError::InvalidPayloadFork)?, - // TODO(electra): finish consolidations when they're more spec'd out - consolidations: Vec::new().into(), }, }), maybe_blobs_and_proofs, diff --git a/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs b/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs index a0c044219d5..74477f5e481 100644 --- a/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs +++ b/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs @@ -171,7 +171,6 @@ where self.include_exits(block)?; self.include_sync_aggregate(block)?; self.include_bls_to_execution_changes(block)?; - self.include_consolidations(block)?; Ok(()) } @@ -360,27 +359,6 @@ where Ok(()) } - /// Includes all signatures in `self.block.body.consolidations` for verification. - pub fn include_consolidations>( - &mut self, - block: &'a SignedBeaconBlock, - ) -> Result<()> { - if let Ok(consolidations) = block.message().body().consolidations() { - self.sets.sets.reserve(consolidations.len()); - for consolidation in consolidations { - let set = consolidation_signature_set( - self.state, - self.get_pubkey.clone(), - consolidation, - self.spec, - )?; - - self.sets.push(set); - } - } - Ok(()) - } - /// Verify all the signatures that have been included in `self`, returning `true` if and only if /// all the signatures are valid. /// diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 06450922fd7..4facf91e1c1 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -4,7 +4,6 @@ use crate::common::{ slash_validator, }; use crate::per_block_processing::errors::{BlockProcessingError, IntoWithIndex}; -use crate::signature_sets::consolidation_signature_set; use crate::VerifySignatures; use types::consts::altair::{PARTICIPATION_FLAG_WEIGHTS, PROPOSER_WEIGHT, WEIGHT_DENOMINATOR}; use types::typenum::U33; @@ -48,7 +47,8 @@ pub fn process_operations>( if let Some(receipts) = receipts { process_deposit_receipts(state, &receipts, spec)?; } - process_consolidations(state, block_body.consolidations()?, verify_signatures, spec)?; + todo!("implement process_consolidation_requests") + //process_consolidation_requests(state, block_body.consolidations()?, verify_signatures, spec)?; } Ok(()) @@ -647,149 +647,10 @@ pub fn process_deposit_receipts( Ok(()) } -pub fn process_consolidations( +pub fn process_consolidation_requests( state: &mut BeaconState, - consolidations: &[SignedConsolidation], - verify_signatures: VerifySignatures, + consolidation_requests: &[ConsolidationRequest], spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { - if consolidations.is_empty() { - return Ok(()); - } - - // If the pending consolidations queue is full, no consolidations are allowed in the block - let pending_consolidations = state.pending_consolidations()?.len(); - let pending_consolidations_limit = E::pending_consolidations_limit(); - block_verify! { - pending_consolidations < pending_consolidations_limit, - BlockProcessingError::TooManyPendingConsolidations { - consolidations: pending_consolidations, - limit: pending_consolidations_limit - } - } - - // If there is too little available consolidation churn limit, no consolidations are allowed in the block - let churn_limit = state.get_consolidation_churn_limit(spec)?; - block_verify! { - churn_limit > spec.min_activation_balance, - BlockProcessingError::ConsolidationChurnLimitTooLow { - churn_limit, - minimum: spec.min_activation_balance - } - } - - for signed_consolidation in consolidations { - let consolidation = signed_consolidation.message.clone(); - - // Verify that source != target, so a consolidation cannot be used as an exit. - block_verify! { - consolidation.source_index != consolidation.target_index, - BlockProcessingError::MatchingSourceTargetConsolidation { - index: consolidation.source_index - } - } - - let source_validator = state.get_validator(consolidation.source_index as usize)?; - let target_validator = state.get_validator(consolidation.target_index as usize)?; - - // Verify the source and the target are active - let current_epoch = state.current_epoch(); - block_verify! { - source_validator.is_active_at(current_epoch), - BlockProcessingError::InactiveConsolidationSource{ - index: consolidation.source_index, - current_epoch - } - } - block_verify! { - target_validator.is_active_at(current_epoch), - BlockProcessingError::InactiveConsolidationTarget{ - index: consolidation.target_index, - current_epoch - } - } - - // Verify exits for source and target have not been initiated - block_verify! { - source_validator.exit_epoch == spec.far_future_epoch, - BlockProcessingError::SourceValidatorExiting{ - index: consolidation.source_index, - } - } - block_verify! { - target_validator.exit_epoch == spec.far_future_epoch, - BlockProcessingError::TargetValidatorExiting{ - index: consolidation.target_index, - } - } - - // Consolidations must specify an epoch when they become valid; they are not valid before then - block_verify! { - current_epoch >= consolidation.epoch, - BlockProcessingError::FutureConsolidationEpoch { - current_epoch, - consolidation_epoch: consolidation.epoch - } - } - - // Verify the source and the target have Execution layer withdrawal credentials - block_verify! { - source_validator.has_execution_withdrawal_credential(spec), - BlockProcessingError::NoSourceExecutionWithdrawalCredential { - index: consolidation.source_index, - } - } - block_verify! { - target_validator.has_execution_withdrawal_credential(spec), - BlockProcessingError::NoTargetExecutionWithdrawalCredential { - index: consolidation.target_index, - } - } - - // Verify the same withdrawal address - let source_address = source_validator - .get_execution_withdrawal_address(spec) - .ok_or(BeaconStateError::NonExecutionAddresWithdrawalCredential)?; - let target_address = target_validator - .get_execution_withdrawal_address(spec) - .ok_or(BeaconStateError::NonExecutionAddresWithdrawalCredential)?; - block_verify! { - source_address == target_address, - BlockProcessingError::MismatchedWithdrawalCredentials { - source_address, - target_address - } - } - - if verify_signatures.is_true() { - let signature_set = consolidation_signature_set( - state, - |i| get_pubkey_from_state(state, i), - signed_consolidation, - spec, - )?; - block_verify! { - signature_set.verify(), - BlockProcessingError::InavlidConsolidationSignature - } - } - let exit_epoch = state.compute_consolidation_epoch_and_update_churn( - source_validator.effective_balance, - spec, - )?; - let source_validator = state.get_validator_mut(consolidation.source_index as usize)?; - // Initiate source validator exit and append pending consolidation - source_validator.exit_epoch = exit_epoch; - source_validator.withdrawable_epoch = source_validator - .exit_epoch - .safe_add(spec.min_validator_withdrawability_delay)?; - state - .pending_consolidations_mut()? - .push(PendingConsolidation { - source_index: consolidation.source_index, - target_index: consolidation.target_index, - })?; - } - - Ok(()) + todo!("implement process_consolidation_requests"); } diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index 3c683766adb..2e00ee03418 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -11,8 +11,8 @@ use types::{ BeaconStateError, ChainSpec, DepositData, Domain, Epoch, EthSpec, Fork, Hash256, InconsistentFork, IndexedAttestation, IndexedAttestationRef, ProposerSlashing, PublicKey, PublicKeyBytes, Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockHeader, - SignedBlsToExecutionChange, SignedConsolidation, SignedContributionAndProof, SignedRoot, - SignedVoluntaryExit, SigningData, Slot, SyncAggregate, SyncAggregatorSelectionData, Unsigned, + SignedBlsToExecutionChange, SignedContributionAndProof, SignedRoot, SignedVoluntaryExit, + SigningData, Slot, SyncAggregate, SyncAggregatorSelectionData, Unsigned, }; pub type Result = std::result::Result; @@ -664,37 +664,3 @@ where message, ))) } - -/// Returns two signature sets, one for the source and one for the target validator -/// in the `SignedConsolidation`. -pub fn consolidation_signature_set<'a, E, F>( - state: &'a BeaconState, - get_pubkey: F, - consolidation: &'a SignedConsolidation, - spec: &'a ChainSpec, -) -> Result> -where - E: EthSpec, - F: Fn(usize) -> Option>, -{ - let source_index = consolidation.message.source_index as usize; - let target_index = consolidation.message.target_index as usize; - - let domain = spec.compute_domain( - Domain::Consolidation, - spec.genesis_fork_version, - state.genesis_validators_root(), - ); - - let message = consolidation.message.signing_root(domain); - let source_pubkey = - get_pubkey(source_index).ok_or(Error::ValidatorUnknown(source_index as u64))?; - let target_pubkey = - get_pubkey(target_index).ok_or(Error::ValidatorUnknown(target_index as u64))?; - - Ok(SignatureSet::multiple_pubkeys( - &consolidation.signature, - vec![source_pubkey, target_pubkey], - message, - )) -} diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index f67a965955c..300faf32f22 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -670,7 +670,6 @@ impl> BeaconBlockElectra graffiti: Graffiti::default(), execution_payload: Payload::Electra::default(), blob_kzg_commitments: VariableList::empty(), - consolidations: VariableList::empty(), }, } } @@ -701,7 +700,6 @@ impl> EmptyBlock for BeaconBlockElec execution_payload: Payload::Electra::default(), bls_to_execution_changes: VariableList::empty(), blob_kzg_commitments: VariableList::empty(), - consolidations: VariableList::empty(), }, } } diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index 199f6c1c8b1..d9cda21cd61 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -114,8 +114,6 @@ pub struct BeaconBlockBody = FullPay VariableList, #[superstruct(only(Deneb, Electra))] pub blob_kzg_commitments: KzgCommitments, - #[superstruct(only(Electra))] - pub consolidations: VariableList, #[superstruct(only(Base, Altair))] #[metastruct(exclude_from(fields))] #[ssz(skip_serializing, skip_deserializing)] @@ -624,7 +622,6 @@ impl From>> execution_payload: FullPayloadElectra { execution_payload }, bls_to_execution_changes, blob_kzg_commitments, - consolidations, } = body; ( @@ -643,7 +640,6 @@ impl From>> }, bls_to_execution_changes, blob_kzg_commitments: blob_kzg_commitments.clone(), - consolidations, }, Some(execution_payload), ) @@ -782,7 +778,6 @@ impl BeaconBlockBodyElectra> { execution_payload: FullPayloadElectra { execution_payload }, bls_to_execution_changes, blob_kzg_commitments, - consolidations, } = self; BeaconBlockBodyElectra { @@ -800,7 +795,6 @@ impl BeaconBlockBodyElectra> { }, bls_to_execution_changes: bls_to_execution_changes.clone(), blob_kzg_commitments: blob_kzg_commitments.clone(), - consolidations: consolidations.clone(), } } } diff --git a/consensus/types/src/consolidation_request.rs b/consensus/types/src/consolidation_request.rs index 7f185a3827b..b21f34e7bba 100644 --- a/consensus/types/src/consolidation_request.rs +++ b/consensus/types/src/consolidation_request.rs @@ -1,5 +1,4 @@ -use crate::Epoch; -use crate::{test_utils::TestRandom, SignedRoot}; +use crate::{test_utils::TestRandom, Address, PublicKeyBytes, SignedRoot}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -20,11 +19,9 @@ use tree_hash_derive::TreeHash; TestRandom, )] pub struct ConsolidationRequest { - #[serde(with = "serde_utils::quoted_u64")] - pub source_index: u64, - #[serde(with = "serde_utils::quoted_u64")] - pub target_index: u64, - pub epoch: Epoch, + pub source_address: Address, + pub source_pubkey: PublicKeyBytes, + pub target_pubkey: PublicKeyBytes, } impl SignedRoot for ConsolidationRequest {} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index cf3e209fad6..769093e5978 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -66,7 +66,6 @@ pub mod signed_aggregate_and_proof; pub mod signed_beacon_block; pub mod signed_beacon_block_header; pub mod signed_bls_to_execution_change; -pub mod signed_consolidation; pub mod signed_contribution_and_proof; pub mod signed_voluntary_exit; pub mod signing_data; @@ -229,7 +228,6 @@ pub use crate::signed_beacon_block::{ }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; -pub use crate::signed_consolidation::SignedConsolidation; pub use crate::signed_contribution_and_proof::SignedContributionAndProof; pub use crate::signed_voluntary_exit::SignedVoluntaryExit; pub use crate::signing_data::{SignedRoot, SigningData}; diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index a22df49ad7b..4d3279a7f77 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -498,7 +498,6 @@ impl SignedBeaconBlockElectra> { execution_payload: BlindedPayloadElectra { .. }, bls_to_execution_changes, blob_kzg_commitments, - consolidations, }, }, signature, @@ -522,7 +521,6 @@ impl SignedBeaconBlockElectra> { execution_payload: FullPayloadElectra { execution_payload }, bls_to_execution_changes, blob_kzg_commitments, - consolidations, }, }, signature, diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index cf684c38e36..80fdeee66e0 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -7,7 +7,8 @@ use ssz::Decode; use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache; use state_processing::epoch_cache::initialize_epoch_cache; use state_processing::per_block_processing::process_operations::{ - process_consolidations, process_deposit_receipts, process_execution_layer_withdrawal_requests, + process_consolidation_requests, process_deposit_receipts, + process_execution_layer_withdrawal_requests, }; use state_processing::{ per_block_processing::{ @@ -26,8 +27,7 @@ use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockBodyBellatrix, BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconState, BlindedPayload, Deposit, DepositRequest, ExecutionPayload, FullPayload, ProposerSlashing, - SignedBlsToExecutionChange, SignedConsolidation, SignedVoluntaryExit, SyncAggregate, - WithdrawalRequest, + SignedBlsToExecutionChange, SignedVoluntaryExit, SyncAggregate, WithdrawalRequest, }; #[derive(Debug, Clone, Default, Deserialize)] @@ -491,29 +491,6 @@ impl Operation for DepositRequest { } } -impl Operation for SignedConsolidation { - fn handler_name() -> String { - "consolidation".into() - } - - fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name >= ForkName::Electra - } - - fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { - ssz_decode_file(path) - } - - fn apply_to( - &self, - state: &mut BeaconState, - spec: &ChainSpec, - _extra: &Operations, - ) -> Result<(), BlockProcessingError> { - process_consolidations(state, &[self.clone()], VerifySignatures::True, spec) - } -} - impl> LoadCase for Operations { fn load_from_dir(path: &Path, fork_name: ForkName) -> Result { let spec = &testing_spec::(fork_name); diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 319e5c49c89..0431ffebd54 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -126,7 +126,6 @@ type_name_generic!(SignedAggregateAndProofBase, "SignedAggregateAndProof"); type_name_generic!(SignedAggregateAndProofElectra, "SignedAggregateAndProof"); type_name_generic!(SignedBeaconBlock); type_name!(SignedBeaconBlockHeader); -type_name!(SignedConsolidation); type_name_generic!(SignedContributionAndProof); type_name!(SignedVoluntaryExit); type_name!(SigningData); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 99028e15458..37699a48a2c 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -107,9 +107,7 @@ fn operations_deposit_requests() { #[test] fn operations_consolidations() { - OperationsHandler::::default().run(); - //TODO(electra): re-enable mainnet once they make tests for this - //OperationsHandler::::default().run(); + todo!("implement once el triggered consolidations are good"); } #[test] @@ -636,12 +634,6 @@ mod ssz_static { SszStaticHandler::::electra_and_later().run(); SszStaticHandler::::electra_and_later().run(); } - - #[test] - fn signed_consolidation() { - SszStaticHandler::::electra_and_later().run(); - SszStaticHandler::::electra_and_later().run(); - } } #[test] From 1e0a3e6c0a69a3ebdcd5a522b065184a0b0a596c Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Thu, 20 Jun 2024 17:13:20 -0500 Subject: [PATCH 170/227] Round Four --- beacon_node/execution_layer/src/engine_api.rs | 15 ++- .../src/engine_api/json_structures.rs | 9 ++ beacon_node/execution_layer/src/lib.rs | 5 + .../test_utils/execution_block_generator.rs | 2 + .../process_operations.rs | 6 +- consensus/types/src/execution_payload.rs | 5 + .../types/src/execution_payload_header.rs | 4 + consensus/types/src/payload.rs | 92 ++++++++++++++++++- consensus/types/src/signed_consolidation.rs | 32 ------- 9 files changed, 131 insertions(+), 39 deletions(-) delete mode 100644 consensus/types/src/signed_consolidation.rs diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index e0bfb7d49eb..6e313fa7ad4 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -20,11 +20,11 @@ use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use strum::IntoStaticStr; use superstruct::superstruct; -use types::execution_payload::{DepositRequests, WithdrawalRequests}; +use types::execution_payload::{ConsolidationRequests, DepositRequests, WithdrawalRequests}; pub use types::{ - Address, BeaconBlockRef, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, - ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, - Withdrawal, Withdrawals, + Address, BeaconBlockRef, ConsolidationRequest, EthSpec, ExecutionBlockHash, ExecutionPayload, + ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, + Uint256, VariableList, Withdrawal, Withdrawals, }; use types::{ ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, @@ -75,6 +75,7 @@ pub enum Error { UnsupportedForkVariant(String), InvalidClientVersion(String), RlpDecoderError(rlp::DecoderError), + TooManyConsolidationRequests(usize), } impl From for Error { @@ -207,6 +208,9 @@ pub struct ExecutionBlockWithTransactions { pub deposit_requests: Vec, #[superstruct(only(Electra))] pub withdrawal_requests: Vec, + #[superstruct(only(Electra))] + // TODO(electra): I don't think we need a JsonConsolidationRequest here because the bytes should be little-endian but we need to confirm + pub consolidation_requests: Vec, } impl TryFrom> for ExecutionBlockWithTransactions { @@ -324,6 +328,7 @@ impl TryFrom> for ExecutionBlockWithTransactions .into_iter() .map(|withdrawal| withdrawal.into()) .collect(), + consolidation_requests: block.consolidation_requests.to_vec(), }) } }; @@ -666,6 +671,8 @@ impl ExecutionPayloadBodyV1 { excess_blob_gas: header.excess_blob_gas, deposit_requests, withdrawal_requests, + // TODO(electra): gonna need to superstruct this thing to do it properly.. + consolidation_requests: Vec::new().into(), })) } } diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 05754cfdb51..0113bcbb347 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -106,6 +106,9 @@ pub struct JsonExecutionPayload { #[superstruct(only(V4))] pub withdrawal_requests: VariableList, + #[superstruct(only(V4))] + pub consolidation_requests: + VariableList, } impl From> for JsonExecutionPayloadV1 { @@ -220,6 +223,11 @@ impl From> for JsonExecutionPayloadV4 .map(Into::into) .collect::>() .into(), + consolidation_requests: payload + .consolidation_requests + .into_iter() + .collect::>() + .into(), } } } @@ -348,6 +356,7 @@ impl From> for ExecutionPayloadElectra .map(Into::into) .collect::>() .into(), + consolidation_requests: payload.consolidation_requests.into(), } } } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index eaa739d7a5d..ff7ada4debb 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -2010,6 +2010,10 @@ impl ExecutionLayer { .collect(), ) .map_err(ApiError::DeserializeWithdrawalRequests)?; + let n_consolidations = electra_block.consolidation_requests.len(); + let consolidation_requests = + VariableList::new(electra_block.consolidation_requests) + .map_err(|_| ApiError::TooManyConsolidationRequests(n_consolidations))?; ExecutionPayload::Electra(ExecutionPayloadElectra { parent_hash: electra_block.parent_hash, fee_recipient: electra_block.fee_recipient, @@ -2030,6 +2034,7 @@ impl ExecutionLayer { excess_blob_gas: electra_block.excess_blob_gas, deposit_requests, withdrawal_requests, + consolidation_requests: consolidation_requests, }) } }; diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 8619e24a238..216c3b7844f 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -659,8 +659,10 @@ impl ExecutionBlockGenerator { withdrawals: pa.withdrawals.clone().into(), blob_gas_used: 0, excess_blob_gas: 0, + // TODO(electra): consider how to test these fields below deposit_requests: vec![].into(), withdrawal_requests: vec![].into(), + consolidation_requests: vec![].into(), }), _ => unreachable!(), }, diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 4facf91e1c1..ca9eae148a3 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -47,8 +47,10 @@ pub fn process_operations>( if let Some(receipts) = receipts { process_deposit_receipts(state, &receipts, spec)?; } - todo!("implement process_consolidation_requests") - //process_consolidation_requests(state, block_body.consolidations()?, verify_signatures, spec)?; + let consolidations = block_body.execution_payload()?.consolidation_requests()?; + if let Some(consolidations) = consolidations { + process_consolidation_requests(state, &consolidations, spec)?; + } } Ok(()) diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index aa6ad8bc95b..90940fbb9b9 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -17,6 +17,8 @@ pub type DepositRequests = VariableList::MaxDepositRequestsPerPayload>; pub type WithdrawalRequests = VariableList::MaxWithdrawalRequestsPerPayload>; +pub type ConsolidationRequests = + VariableList::MaxConsolidationRequestsPerPayload>; #[superstruct( variants(Bellatrix, Capella, Deneb, Electra), @@ -97,6 +99,9 @@ pub struct ExecutionPayload { pub deposit_requests: VariableList, #[superstruct(only(Electra))] pub withdrawal_requests: VariableList, + #[superstruct(only(Electra))] + pub consolidation_requests: + VariableList, } impl<'a, E: EthSpec> ExecutionPayloadRef<'a, E> { diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index fe94c8024e7..dd8245d8bbd 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -89,6 +89,8 @@ pub struct ExecutionPayloadHeader { pub deposit_requests_root: Hash256, #[superstruct(only(Electra), partial_getter(copy))] pub withdrawal_requests_root: Hash256, + #[superstruct(only(Electra), partial_getter(copy))] + pub consolidation_requests_root: Hash256, } impl ExecutionPayloadHeader { @@ -208,6 +210,7 @@ impl ExecutionPayloadHeaderDeneb { excess_blob_gas: self.excess_blob_gas, deposit_requests_root: Hash256::zero(), withdrawal_requests_root: Hash256::zero(), + consolidation_requests_root: Hash256::zero(), } } } @@ -301,6 +304,7 @@ impl<'a, E: EthSpec> From<&'a ExecutionPayloadElectra> for ExecutionPayloadHe excess_blob_gas: payload.excess_blob_gas, deposit_requests_root: payload.deposit_requests.tree_hash_root(), withdrawal_requests_root: payload.withdrawal_requests.tree_hash_root(), + consolidation_requests_root: payload.consolidation_requests.tree_hash_root(), } } } diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 4f1a8574623..cee8b8cc219 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -45,6 +45,12 @@ pub trait ExecPayload: Debug + Clone + PartialEq + Hash + TreeHash + fn deposit_requests( &self, ) -> Result>, Error>; + fn consolidation_requests( + &self, + ) -> Result< + Option>, + Error, + >; /// Is this a default payload with 0x0 roots for transactions and withdrawals? fn is_default_with_zero_roots(&self) -> bool; @@ -311,6 +317,24 @@ impl ExecPayload for FullPayload { } } + fn consolidation_requests( + &self, + ) -> Result< + Option< + VariableList::MaxConsolidationRequestsPerPayload>, + >, + Error, + > { + match self { + FullPayload::Bellatrix(_) | FullPayload::Capella(_) | FullPayload::Deneb(_) => { + Err(Error::IncorrectStateVariant) + } + FullPayload::Electra(inner) => { + Ok(Some(inner.execution_payload.consolidation_requests.clone())) + } + } + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_full_payload_ref!(&'a _, self.to_ref(), move |payload, cons| { cons(payload); @@ -470,6 +494,24 @@ impl<'b, E: EthSpec> ExecPayload for FullPayloadRef<'b, E> { } } + fn consolidation_requests( + &self, + ) -> Result< + Option< + VariableList::MaxConsolidationRequestsPerPayload>, + >, + Error, + > { + match self { + FullPayloadRef::Bellatrix(_) + | FullPayloadRef::Capella(_) + | FullPayloadRef::Deneb(_) => Err(Error::IncorrectStateVariant), + FullPayloadRef::Electra(inner) => { + Ok(Some(inner.execution_payload.consolidation_requests.clone())) + } + } + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_full_payload_ref!(&'a _, self, move |payload, cons| { cons(payload); @@ -663,6 +705,17 @@ impl ExecPayload for BlindedPayload { Ok(None) } + fn consolidation_requests( + &self, + ) -> Result< + Option< + VariableList::MaxConsolidationRequestsPerPayload>, + >, + Error, + > { + Ok(None) + } + fn is_default_with_zero_roots(&self) -> bool { self.to_ref().is_default_with_zero_roots() } @@ -777,6 +830,17 @@ impl<'b, E: EthSpec> ExecPayload for BlindedPayloadRef<'b, E> { Ok(None) } + fn consolidation_requests( + &self, + ) -> Result< + Option< + VariableList::MaxConsolidationRequestsPerPayload>, + >, + Error, + > { + Ok(None) + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_blinded_payload_ref!(&'b _, self, move |payload, cons| { cons(payload); @@ -805,7 +869,8 @@ macro_rules! impl_exec_payload_common { $g:block, $h:block, $i:block, - $j:block) => { + $j:block, + $k:block) => { impl ExecPayload for $wrapper_type { fn block_type() -> BlockType { BlockType::$block_type_variant @@ -885,6 +950,13 @@ macro_rules! impl_exec_payload_common { let j = $j; j(self) } + + fn consolidation_requests( + &self, + ) -> Result::MaxConsolidationRequestsPerPayload>>, Error> { + let k = $k; + k(self) + } } impl From<$wrapped_type> for $wrapper_type { @@ -932,6 +1004,7 @@ macro_rules! impl_exec_payload_for_fork { c }, { |_| { Ok(None) } }, + { |_| { Ok(None) } }, { |_| { Ok(None) } } ); @@ -1043,6 +1116,23 @@ macro_rules! impl_exec_payload_for_fork { wrapper_ref_type.deposit_requests() }; c + }, + { + let c: for<'a> fn( + &'a $wrapper_type_full, + ) -> Result< + Option< + VariableList< + ConsolidationRequest, + ::MaxConsolidationRequestsPerPayload, + >, + >, + Error, + > = |payload: &$wrapper_type_full| { + let wrapper_ref_type = FullPayloadRef::$fork_variant(&payload); + wrapper_ref_type.consolidation_requests() + }; + c } ); diff --git a/consensus/types/src/signed_consolidation.rs b/consensus/types/src/signed_consolidation.rs deleted file mode 100644 index c6316995b37..00000000000 --- a/consensus/types/src/signed_consolidation.rs +++ /dev/null @@ -1,32 +0,0 @@ -use crate::test_utils::TestRandom; -use crate::{ConsolidationRequest, Signature}; -use serde::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; -use tree_hash_derive::TreeHash; - -#[derive( - arbitrary::Arbitrary, - Debug, - PartialEq, - Eq, - Hash, - Clone, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, -)] -pub struct SignedConsolidation { - pub message: ConsolidationRequest, - pub signature: Signature, -} - -#[cfg(test)] -mod tests { - use super::*; - - ssz_and_tree_hash_tests!(SignedConsolidation); -} From 27ed90e4dc88b61c7e7c190d088560baf2412362 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 21 Jun 2024 00:20:10 -0400 Subject: [PATCH 171/227] Electra attestation changes sean review (#5972) * instantiate empty bitlist in unreachable code * clean up error conversion * fork enabled bool cleanup * remove a couple todos * return bools instead of options in `aggregate` and use the result * delete commented out code * use map macros in simple transformations * remove signers_disjoint_from * get ef tests compiling * get ef tests compiling * update intentionally excluded files --- .../src/attestation_verification.rs | 32 +++---------------- beacon_node/operation_pool/src/attestation.rs | 4 +-- .../operation_pool/src/attestation_storage.rs | 24 +++++++------- beacon_node/operation_pool/src/lib.rs | 8 ++--- .../state_processing/src/verify_operation.rs | 1 - consensus/types/src/aggregate_and_proof.rs | 19 ++++++----- consensus/types/src/attestation.rs | 23 ++----------- consensus/types/src/fork_name.rs | 4 +++ .../types/src/signed_aggregate_and_proof.rs | 24 +++++++------- .../types/src/sync_committee_contribution.rs | 8 ----- testing/ef_tests/check_all_files_accessed.py | 4 ++- testing/ef_tests/src/type_name.rs | 4 +++ testing/ef_tests/tests/tests.rs | 1 - 13 files changed, 56 insertions(+), 100 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 32c33f3e617..c7f7bb58dab 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -61,10 +61,9 @@ use std::borrow::Cow; use strum::AsRefStr; use tree_hash::TreeHash; use types::{ - Attestation, AttestationRef, BeaconCommittee, - BeaconStateError::{self, NoCommitteeFound}, - ChainSpec, CommitteeIndex, Epoch, EthSpec, ForkName, Hash256, IndexedAttestation, - SelectionProof, SignedAggregateAndProof, Slot, SubnetId, + Attestation, AttestationRef, BeaconCommittee, BeaconStateError::NoCommitteeFound, ChainSpec, + CommitteeIndex, Epoch, EthSpec, Hash256, IndexedAttestation, SelectionProof, + SignedAggregateAndProof, Slot, SubnetId, }; pub use batch::{batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations}; @@ -266,30 +265,9 @@ pub enum Error { BeaconChainError(BeaconChainError), } -// TODO(electra) the error conversion changes here are to get a test case to pass -// this could easily be cleaned up impl From for Error { fn from(e: BeaconChainError) -> Self { - match &e { - BeaconChainError::BeaconStateError(beacon_state_error) => { - if let BeaconStateError::AggregatorNotInCommittee { aggregator_index } = - beacon_state_error - { - Self::AggregatorNotInCommittee { - aggregator_index: *aggregator_index, - } - } else if let BeaconStateError::InvalidSelectionProof { aggregator_index } = - beacon_state_error - { - Self::InvalidSelectionProof { - aggregator_index: *aggregator_index, - } - } else { - Error::BeaconChainError(e) - } - } - _ => Error::BeaconChainError(e), - } + Self::BeaconChainError(e) } } @@ -1169,7 +1147,7 @@ pub fn verify_propagation_slot_range( let current_fork = spec.fork_name_at_slot::(slot_clock.now().ok_or(BeaconChainError::UnableToReadSlot)?); - let earliest_permissible_slot = if current_fork < ForkName::Deneb { + let earliest_permissible_slot = if !current_fork.deneb_enabled() { one_epoch_prior // EIP-7045 } else { diff --git a/beacon_node/operation_pool/src/attestation.rs b/beacon_node/operation_pool/src/attestation.rs index c6ed6eb7f6e..97d0583e345 100644 --- a/beacon_node/operation_pool/src/attestation.rs +++ b/beacon_node/operation_pool/src/attestation.rs @@ -184,8 +184,8 @@ pub fn earliest_attestation_validators( // Bitfield of validators whose attestations are new/fresh. let mut new_validators = match attestation.indexed { CompactIndexedAttestation::Base(indexed_att) => indexed_att.aggregation_bits.clone(), - // TODO(electra) per the comments above, this code path is obsolete post altair fork, so maybe we should just return an empty bitlist here? - CompactIndexedAttestation::Electra(_) => todo!(), + // This code path is obsolete post altair fork, so we just return an empty bitlist here. + CompactIndexedAttestation::Electra(_) => return BitList::with_capacity(0).unwrap(), }; let state_attestations = if attestation.checkpoint.target_epoch == state.current_epoch() { diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index ad4ffe6d3c1..4de9d351f3c 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -165,22 +165,22 @@ impl CompactIndexedAttestation { CompactIndexedAttestation::Electra(this), CompactIndexedAttestation::Electra(other), ) => this.should_aggregate(other), - // TODO(electra) is a mix of electra and base compact indexed attestations an edge case we need to deal with? _ => false, } } - pub fn aggregate(&mut self, other: &Self) -> Option<()> { + /// Returns `true` if aggregated, otherwise `false`. + pub fn aggregate(&mut self, other: &Self) -> bool { match (self, other) { (CompactIndexedAttestation::Base(this), CompactIndexedAttestation::Base(other)) => { - this.aggregate(other) + this.aggregate(other); + true } ( CompactIndexedAttestation::Electra(this), CompactIndexedAttestation::Electra(other), ) => this.aggregate_same_committee(other), - // TODO(electra) is a mix of electra and base compact indexed attestations an edge case we need to deal with? - _ => None, + _ => false, } } } @@ -192,7 +192,7 @@ impl CompactIndexedAttestationBase { .is_zero() } - pub fn aggregate(&mut self, other: &Self) -> Option<()> { + pub fn aggregate(&mut self, other: &Self) { self.attesting_indices = self .attesting_indices .drain(..) @@ -201,8 +201,6 @@ impl CompactIndexedAttestationBase { .collect(); self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); self.signature.add_assign_aggregate(&other.signature); - - Some(()) } } @@ -216,9 +214,10 @@ impl CompactIndexedAttestationElectra { .is_zero() } - pub fn aggregate_same_committee(&mut self, other: &Self) -> Option<()> { + /// Returns `true` if aggregated, otherwise `false`. + pub fn aggregate_same_committee(&mut self, other: &Self) -> bool { if self.committee_bits != other.committee_bits { - return None; + return false; } self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); self.attesting_indices = self @@ -228,7 +227,7 @@ impl CompactIndexedAttestationElectra { .dedup() .collect(); self.signature.add_assign_aggregate(&other.signature); - Some(()) + true } pub fn aggregate_with_disjoint_committees(&mut self, other: &Self) -> Option<()> { @@ -318,8 +317,7 @@ impl AttestationMap { for existing_attestation in attestations.iter_mut() { if existing_attestation.should_aggregate(&indexed) { - existing_attestation.aggregate(&indexed); - aggregated = true; + aggregated = existing_attestation.aggregate(&indexed); } else if *existing_attestation == indexed { aggregated = true; } diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 49fbed56869..a1c9ada03a0 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -39,7 +39,7 @@ use std::ptr; use types::{ sync_aggregate::Error as SyncAggregateError, typenum::Unsigned, AbstractExecPayload, Attestation, AttestationData, AttesterSlashing, BeaconState, BeaconStateError, ChainSpec, - Epoch, EthSpec, ForkName, ProposerSlashing, SignedBeaconBlock, SignedBlsToExecutionChange, + Epoch, EthSpec, ProposerSlashing, SignedBeaconBlock, SignedBlsToExecutionChange, SignedVoluntaryExit, Slot, SyncAggregate, SyncCommitteeContribution, Validator, }; @@ -316,10 +316,10 @@ impl OperationPool { ) .inspect(|_| num_curr_valid += 1); - let curr_epoch_limit = if fork_name < ForkName::Electra { - E::MaxAttestations::to_usize() - } else { + let curr_epoch_limit = if fork_name.electra_enabled() { E::MaxAttestationsElectra::to_usize() + } else { + E::MaxAttestations::to_usize() }; let prev_epoch_limit = if let BeaconState::Base(base_state) = state { std::cmp::min( diff --git a/consensus/state_processing/src/verify_operation.rs b/consensus/state_processing/src/verify_operation.rs index f218b806d20..c4b7c6a026f 100644 --- a/consensus/state_processing/src/verify_operation.rs +++ b/consensus/state_processing/src/verify_operation.rs @@ -48,7 +48,6 @@ pub trait TransformPersist { pub struct SigVerifiedOp { op: T, verified_against: VerifiedAgainst, - //#[ssz(skip_serializing, skip_deserializing)] _phantom: PhantomData, } diff --git a/consensus/types/src/aggregate_and_proof.rs b/consensus/types/src/aggregate_and_proof.rs index 6ae0df4ad7c..223b12e7684 100644 --- a/consensus/types/src/aggregate_and_proof.rs +++ b/consensus/types/src/aggregate_and_proof.rs @@ -33,7 +33,8 @@ use tree_hash_derive::TreeHash; derive(Debug, PartialEq, TreeHash, Serialize,), serde(untagged, bound = "E: EthSpec"), tree_hash(enum_behaviour = "transparent") - ) + ), + map_ref_into(AttestationRef) )] #[derive( arbitrary::Arbitrary, Debug, Clone, PartialEq, Serialize, Deserialize, Encode, TreeHash, @@ -59,19 +60,17 @@ pub struct AggregateAndProof { impl<'a, E: EthSpec> AggregateAndProofRef<'a, E> { /// Returns `true` if `validator_pubkey` signed over `self.aggregate.data.slot`. pub fn aggregate(self) -> AttestationRef<'a, E> { - match self { - AggregateAndProofRef::Base(a) => AttestationRef::Base(&a.aggregate), - AggregateAndProofRef::Electra(a) => AttestationRef::Electra(&a.aggregate), - } + map_aggregate_and_proof_ref_into_attestation_ref!(&'a _, self, |inner, cons| { + cons(&inner.aggregate) + }) } } impl AggregateAndProof { /// Returns `true` if `validator_pubkey` signed over `self.aggregate.data.slot`. - pub fn aggregate(&self) -> AttestationRef { - match self { - AggregateAndProof::Base(a) => AttestationRef::Base(&a.aggregate), - AggregateAndProof::Electra(a) => AttestationRef::Electra(&a.aggregate), - } + pub fn aggregate<'a>(&'a self) -> AttestationRef<'a, E> { + map_aggregate_and_proof_ref_into_attestation_ref!(&'a _, self.to_ref(), |inner, cons| { + cons(&inner.aggregate) + }) } } diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index c8d1c3fb9b7..88993267a94 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -1,6 +1,6 @@ use crate::slot_data::SlotData; +use crate::Checkpoint; use crate::{test_utils::TestRandom, Hash256, Slot}; -use crate::{Checkpoint, ForkName}; use derivative::Derivative; use safe_arith::ArithError; use serde::{Deserialize, Serialize}; @@ -99,7 +99,7 @@ impl Attestation { target: Checkpoint, spec: &ChainSpec, ) -> Result { - if spec.fork_name_at_slot::(slot) >= ForkName::Electra { + if spec.fork_name_at_slot::(slot).electra_enabled() { let mut committee_bits: BitVector = BitVector::default(); committee_bits .set(committee_index as usize, true) @@ -277,16 +277,6 @@ impl<'a, E: EthSpec> AttestationRef<'a, E> { } impl AttestationElectra { - /// Are the aggregation bitfields of these attestations disjoint? - // TODO(electra): check whether the definition from CompactIndexedAttestation::should_aggregate - // is useful where this is used, i.e. only consider attestations disjoint when their committees - // match AND their aggregation bits do not intersect. - pub fn signers_disjoint_from(&self, other: &Self) -> bool { - self.aggregation_bits - .intersection(&other.aggregation_bits) - .is_zero() - } - pub fn committee_index(&self) -> Option { self.get_committee_indices().first().cloned() } @@ -304,7 +294,6 @@ impl AttestationElectra { /// The aggregation bitfields must be disjoint, and the data must be the same. pub fn aggregate(&mut self, other: &Self) { debug_assert_eq!(self.data, other.data); - debug_assert!(self.signers_disjoint_from(other)); self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); self.signature.add_assign_aggregate(&other.signature); } @@ -358,19 +347,11 @@ impl AttestationElectra { } impl AttestationBase { - /// Are the aggregation bitfields of these attestations disjoint? - pub fn signers_disjoint_from(&self, other: &Self) -> bool { - self.aggregation_bits - .intersection(&other.aggregation_bits) - .is_zero() - } - /// Aggregate another Attestation into this one. /// /// The aggregation bitfields must be disjoint, and the data must be the same. pub fn aggregate(&mut self, other: &Self) { debug_assert_eq!(self.data, other.data); - debug_assert!(self.signers_disjoint_from(other)); self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); self.signature.add_assign_aggregate(&other.signature); } diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index f0810e2bdbe..96f206d4543 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -120,6 +120,10 @@ impl ForkName { } } + pub fn deneb_enabled(self) -> bool { + self >= ForkName::Deneb + } + pub fn electra_enabled(self) -> bool { self >= ForkName::Electra } diff --git a/consensus/types/src/signed_aggregate_and_proof.rs b/consensus/types/src/signed_aggregate_and_proof.rs index d339cecaae0..26eca19bf15 100644 --- a/consensus/types/src/signed_aggregate_and_proof.rs +++ b/consensus/types/src/signed_aggregate_and_proof.rs @@ -34,7 +34,9 @@ use tree_hash_derive::TreeHash; ), serde(bound = "E: EthSpec"), arbitrary(bound = "E: EthSpec"), - ) + ), + map_into(Attestation), + map_ref_into(AggregateAndProofRef) )] #[derive( arbitrary::Arbitrary, Debug, Clone, PartialEq, Serialize, Deserialize, Encode, TreeHash, @@ -102,19 +104,17 @@ impl SignedAggregateAndProof { } } - pub fn message(&self) -> AggregateAndProofRef { - match self { - SignedAggregateAndProof::Base(message) => AggregateAndProofRef::Base(&message.message), - SignedAggregateAndProof::Electra(message) => { - AggregateAndProofRef::Electra(&message.message) - } - } + pub fn message<'a>(&'a self) -> AggregateAndProofRef<'a, E> { + map_signed_aggregate_and_proof_ref_into_aggregate_and_proof_ref!( + &'a _, + self.to_ref(), + |inner, cons| { cons(&inner.message) } + ) } pub fn into_attestation(self) -> Attestation { - match self { - Self::Base(att) => Attestation::Base(att.message.aggregate), - Self::Electra(att) => Attestation::Electra(att.message.aggregate), - } + map_signed_aggregate_and_proof_into_attestation!(self, |inner, cons| { + cons(inner.message.aggregate) + }) } } diff --git a/consensus/types/src/sync_committee_contribution.rs b/consensus/types/src/sync_committee_contribution.rs index 81de83ba3be..c348c3e8be3 100644 --- a/consensus/types/src/sync_committee_contribution.rs +++ b/consensus/types/src/sync_committee_contribution.rs @@ -63,13 +63,6 @@ impl SyncCommitteeContribution { }) } - /// Are the aggregation bitfields of these sync contribution disjoint? - pub fn signers_disjoint_from(&self, other: &Self) -> bool { - self.aggregation_bits - .intersection(&other.aggregation_bits) - .is_zero() - } - /// Aggregate another `SyncCommitteeContribution` into this one. /// /// The aggregation bitfields must be disjoint, and the data must be the same. @@ -77,7 +70,6 @@ impl SyncCommitteeContribution { debug_assert_eq!(self.slot, other.slot); debug_assert_eq!(self.beacon_block_root, other.beacon_block_root); debug_assert_eq!(self.subcommittee_index, other.subcommittee_index); - debug_assert!(self.signers_disjoint_from(other)); self.aggregation_bits = self.aggregation_bits.union(&other.aggregation_bits); self.signature.add_assign_aggregate(&other.signature); diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index 7629d61827f..5f425ab98b4 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -42,7 +42,9 @@ "bls12-381-tests/deserialization_G2", "bls12-381-tests/hash_to_G2", "tests/.*/eip6110", - "tests/.*/whisk" + "tests/.*/whisk", + # TODO(electra): re-enable in https://github.com/sigp/lighthouse/pull/5758 + "tests/.*/.*/ssz_static/IndexedAttestation" ] diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index cbea78dabfc..f886431b400 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -41,6 +41,8 @@ type_name_generic!(AggregateAndProof); type_name_generic!(AggregateAndProofBase, "AggregateAndProof"); type_name_generic!(AggregateAndProofElectra, "AggregateAndProof"); type_name_generic!(Attestation); +type_name_generic!(AttestationBase, "Attestation"); +type_name_generic!(AttestationElectra, "Attestation"); type_name!(AttestationData); type_name_generic!(AttesterSlashing); type_name_generic!(AttesterSlashingBase, "AttesterSlashing"); @@ -76,6 +78,8 @@ type_name!(Fork); type_name!(ForkData); type_name_generic!(HistoricalBatch); type_name_generic!(IndexedAttestation); +type_name_generic!(IndexedAttestationBase, "IndexedAttestation"); +type_name_generic!(IndexedAttestationElectra, "IndexedAttestation"); type_name_generic!(LightClientBootstrap); type_name_generic!(LightClientBootstrapAltair, "LightClientBootstrap"); type_name_generic!(LightClientBootstrapCapella, "LightClientBootstrap"); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 9e8b375b2c1..60685140100 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -232,7 +232,6 @@ mod ssz_static { ssz_static_test!(fork, Fork); ssz_static_test!(fork_data, ForkData); ssz_static_test!(historical_batch, HistoricalBatch<_>); - ssz_static_test!(indexed_attestation, IndexedAttestation<_>); ssz_static_test!(pending_attestation, PendingAttestation<_>); ssz_static_test!(proposer_slashing, ProposerSlashing); ssz_static_test!( From b6913ae542e163f2d688afa65549cd79f1c5d06a Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 21 Jun 2024 12:02:23 +1000 Subject: [PATCH 172/227] Avoid changing slasher schema for Electra --- Cargo.lock | 1 + beacon_node/src/lib.rs | 10 +- consensus/types/src/indexed_attestation.rs | 32 ----- consensus/types/src/lib.rs | 3 +- slasher/Cargo.toml | 1 + slasher/src/database.rs | 137 +++++++++++++++++++-- slasher/src/error.rs | 7 ++ slasher/src/slasher.rs | 7 +- slasher/src/test_utils.rs | 9 +- slasher/tests/attester_slashings.rs | 9 +- slasher/tests/proposer_slashings.rs | 8 +- slasher/tests/random.rs | 6 +- slasher/tests/wrap_around.rs | 8 +- 13 files changed, 178 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 753763fe96e..90196ea5b31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7653,6 +7653,7 @@ dependencies = [ "safe_arith", "serde", "slog", + "ssz_types", "strum", "tempfile", "tree_hash", diff --git a/beacon_node/src/lib.rs b/beacon_node/src/lib.rs index ab400d2e730..385da5b4fe1 100644 --- a/beacon_node/src/lib.rs +++ b/beacon_node/src/lib.rs @@ -80,7 +80,7 @@ impl ProductionBeaconNode { let builder = ClientBuilder::new(context.eth_spec_instance.clone()) .runtime_context(context) - .chain_spec(spec) + .chain_spec(spec.clone()) .beacon_processor(client_config.beacon_processor.clone()) .http_api_config(client_config.http_api.clone()) .disk_store( @@ -113,8 +113,12 @@ impl ProductionBeaconNode { _ => {} } let slasher = Arc::new( - Slasher::open(slasher_config, log.new(slog::o!("service" => "slasher"))) - .map_err(|e| format!("Slasher open error: {:?}", e))?, + Slasher::open( + slasher_config, + Arc::new(spec), + log.new(slog::o!("service" => "slasher")), + ) + .map_err(|e| format!("Slasher open error: {:?}", e))?, ); builder.slasher(slasher) } else { diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index 900c9059389..19d15010753 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -240,38 +240,6 @@ mod quoted_variable_list_u64 { } } -#[derive(Debug, Clone, Encode, Decode, PartialEq)] -#[ssz(enum_behaviour = "union")] -pub enum IndexedAttestationOnDisk { - Base(IndexedAttestationBase), - Electra(IndexedAttestationElectra), -} - -#[derive(Debug, Clone, Encode, PartialEq)] -#[ssz(enum_behaviour = "union")] -pub enum IndexedAttestationRefOnDisk<'a, E: EthSpec> { - Base(&'a IndexedAttestationBase), - Electra(&'a IndexedAttestationElectra), -} - -impl<'a, E: EthSpec> From<&'a IndexedAttestation> for IndexedAttestationRefOnDisk<'a, E> { - fn from(attestation: &'a IndexedAttestation) -> Self { - match attestation { - IndexedAttestation::Base(attestation) => Self::Base(attestation), - IndexedAttestation::Electra(attestation) => Self::Electra(attestation), - } - } -} - -impl From> for IndexedAttestation { - fn from(attestation: IndexedAttestationOnDisk) -> Self { - match attestation { - IndexedAttestationOnDisk::Base(attestation) => Self::Base(attestation), - IndexedAttestationOnDisk::Electra(attestation) => Self::Electra(attestation), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 2a73ff0f37a..75456446bf2 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -176,8 +176,7 @@ pub use crate::fork_versioned_response::{ForkVersionDeserialize, ForkVersionedRe pub use crate::graffiti::{Graffiti, GRAFFITI_BYTES_LEN}; pub use crate::historical_batch::HistoricalBatch; pub use crate::indexed_attestation::{ - IndexedAttestation, IndexedAttestationBase, IndexedAttestationElectra, - IndexedAttestationOnDisk, IndexedAttestationRef, IndexedAttestationRefOnDisk, + IndexedAttestation, IndexedAttestationBase, IndexedAttestationElectra, IndexedAttestationRef, }; pub use crate::light_client_bootstrap::{ LightClientBootstrap, LightClientBootstrapAltair, LightClientBootstrapCapella, diff --git a/slasher/Cargo.toml b/slasher/Cargo.toml index ef5cb8249ec..563c4599d8b 100644 --- a/slasher/Cargo.toml +++ b/slasher/Cargo.toml @@ -29,6 +29,7 @@ tree_hash = { workspace = true } tree_hash_derive = { workspace = true } types = { workspace = true } strum = { workspace = true } +ssz_types = { workspace = true } # MDBX is pinned at the last version with Windows and macOS support. mdbx = { package = "libmdbx", git = "https://github.com/sigp/libmdbx-rs", tag = "v0.1.4", optional = true } diff --git a/slasher/src/database.rs b/slasher/src/database.rs index 16089706a00..2463c77f41e 100644 --- a/slasher/src/database.rs +++ b/slasher/src/database.rs @@ -13,13 +13,15 @@ use parking_lot::Mutex; use serde::de::DeserializeOwned; use slog::{info, Logger}; use ssz::{Decode, Encode}; +use ssz_derive::{Decode, Encode}; use std::borrow::{Borrow, Cow}; use std::marker::PhantomData; use std::sync::Arc; use tree_hash::TreeHash; use types::{ - Epoch, EthSpec, Hash256, IndexedAttestation, IndexedAttestationOnDisk, - IndexedAttestationRefOnDisk, ProposerSlashing, SignedBeaconBlockHeader, Slot, + AggregateSignature, AttestationData, ChainSpec, Epoch, EthSpec, ForkName, Hash256, + IndexedAttestation, IndexedAttestationBase, IndexedAttestationElectra, ProposerSlashing, + SignedBeaconBlockHeader, Slot, VariableList, }; /// Current database schema version, to check compatibility of on-disk DB with software. @@ -70,6 +72,7 @@ pub struct SlasherDB { /// LRU cache mapping indexed attestation IDs to their attestation data roots. attestation_root_cache: Mutex>, pub(crate) config: Arc, + pub(crate) spec: Arc, _phantom: PhantomData, } @@ -236,6 +239,43 @@ impl AsRef<[u8]> for IndexedAttestationId { } } +/// Indexed attestation that abstracts over Phase0 and Electra variants by using a plain `Vec` for +/// the attesting indices. +/// +/// This allows us to avoid rewriting the entire indexed attestation database at Electra, which +/// saves a lot of execution time. The bytes that it encodes to are the same as the bytes that a +/// regular IndexedAttestation encodes to, because SSZ doesn't care about the length-bound. +#[derive(Debug, PartialEq, Decode, Encode)] +pub struct IndexedAttestationOnDisk { + attesting_indices: Vec, + data: AttestationData, + signature: AggregateSignature, +} + +impl IndexedAttestationOnDisk { + fn into_indexed_attestation( + self, + spec: &ChainSpec, + ) -> Result, Error> { + let fork_at_target_epoch = spec.fork_name_at_epoch(self.data.target.epoch); + if fork_at_target_epoch >= ForkName::Electra { + let attesting_indices = VariableList::new(self.attesting_indices)?; + Ok(IndexedAttestation::Electra(IndexedAttestationElectra { + attesting_indices, + data: self.data, + signature: self.signature, + })) + } else { + let attesting_indices = VariableList::new(self.attesting_indices)?; + Ok(IndexedAttestation::Base(IndexedAttestationBase { + attesting_indices, + data: self.data, + signature: self.signature, + })) + } + } +} + /// Bincode deserialization specialised to `Cow<[u8]>`. fn bincode_deserialize(bytes: Cow<[u8]>) -> Result { Ok(bincode::deserialize(bytes.borrow())?) @@ -246,7 +286,7 @@ fn ssz_decode(bytes: Cow<[u8]>) -> Result { } impl SlasherDB { - pub fn open(config: Arc, log: Logger) -> Result { + pub fn open(config: Arc, spec: Arc, log: Logger) -> Result { info!(log, "Opening slasher database"; "backend" => %config.backend); std::fs::create_dir_all(&config.database_path)?; @@ -269,6 +309,7 @@ impl SlasherDB { databases, attestation_root_cache, config, + spec, _phantom: PhantomData, }; @@ -458,9 +499,8 @@ impl SlasherDB { }; let attestation_key = IndexedAttestationId::new(indexed_att_id); - let indexed_attestation_on_disk: IndexedAttestationRefOnDisk = - indexed_attestation.into(); - let data = indexed_attestation_on_disk.as_ssz_bytes(); + // IndexedAttestationOnDisk and IndexedAttestation have compatible encodings. + let data = indexed_attestation.as_ssz_bytes(); cursor.put(attestation_key.as_ref(), &data)?; drop(cursor); @@ -484,8 +524,8 @@ impl SlasherDB { .ok_or(Error::MissingIndexedAttestation { id: indexed_attestation_id.as_u64(), })?; - let indexed_attestation: IndexedAttestationOnDisk = ssz_decode(bytes)?; - Ok(indexed_attestation.into()) + let indexed_attestation_on_disk: IndexedAttestationOnDisk = ssz_decode(bytes)?; + indexed_attestation_on_disk.into_indexed_attestation(&self.spec) } fn get_attestation_data_root( @@ -775,3 +815,84 @@ impl SlasherDB { Ok(()) } } + +#[cfg(test)] +mod test { + use super::*; + use types::{Checkpoint, MainnetEthSpec}; + + type E = MainnetEthSpec; + + fn indexed_attestation_on_disk_roundtrip_test( + spec: &ChainSpec, + make_attestation: fn( + Vec, + AttestationData, + AggregateSignature, + ) -> IndexedAttestation, + ) { + let attestation_data = AttestationData { + slot: Slot::new(1000), + index: 0, + beacon_block_root: Hash256::repeat_byte(0xaa), + source: Checkpoint { + epoch: Epoch::new(0), + root: Hash256::repeat_byte(0xbb), + }, + target: Checkpoint { + epoch: Epoch::new(31), + root: Hash256::repeat_byte(0xcc), + }, + }; + + let attesting_indices = vec![1, 14, 160, 812737]; + let signature = AggregateSignature::infinity(); + + let fork_attestation = make_attestation( + attesting_indices.clone(), + attestation_data.clone(), + signature.clone(), + ); + + let on_disk = IndexedAttestationOnDisk { + attesting_indices, + data: attestation_data, + signature, + }; + let encoded = on_disk.as_ssz_bytes(); + assert_eq!(encoded, fork_attestation.as_ssz_bytes()); + + let decoded_on_disk = IndexedAttestationOnDisk::from_ssz_bytes(&encoded).unwrap(); + assert_eq!(decoded_on_disk, on_disk); + + let decoded = on_disk.into_indexed_attestation(&spec).unwrap(); + assert_eq!(decoded, fork_attestation); + } + + /// Check that `IndexedAttestationOnDisk` and `IndexedAttestation` have compatible encodings. + #[test] + fn indexed_attestation_on_disk_roundtrip_base() { + let spec = ForkName::Base.make_genesis_spec(E::default_spec()); + let make_attestation = |attesting_indices, data, signature| { + IndexedAttestation::::Base(IndexedAttestationBase { + attesting_indices: VariableList::new(attesting_indices).unwrap(), + data, + signature, + }) + }; + indexed_attestation_on_disk_roundtrip_test(&spec, make_attestation) + } + + #[test] + fn indexed_attestation_on_disk_roundtrip_electra() { + let spec = ForkName::Electra.make_genesis_spec(E::default_spec()); + let make_attestation = |attesting_indices, data, signature| { + IndexedAttestation::::Electra(IndexedAttestationElectra { + attesting_indices: VariableList::new(attesting_indices).unwrap(), + data, + signature, + }) + }; + indexed_attestation_on_disk_roundtrip_test(&spec, make_attestation) + } +} diff --git a/slasher/src/error.rs b/slasher/src/error.rs index b939c281e9f..8d3295b22a8 100644 --- a/slasher/src/error.rs +++ b/slasher/src/error.rs @@ -13,6 +13,7 @@ pub enum Error { DatabaseIOError(io::Error), DatabasePermissionsError(filesystem::Error), SszDecodeError(ssz::DecodeError), + SszTypesError(ssz_types::Error), BincodeError(bincode::Error), ArithError(safe_arith::ArithError), ChunkIndexOutOfBounds(usize), @@ -100,6 +101,12 @@ impl From for Error { } } +impl From for Error { + fn from(e: ssz_types::Error) -> Self { + Error::SszTypesError(e) + } +} + impl From for Error { fn from(e: bincode::Error) -> Self { Error::BincodeError(e) diff --git a/slasher/src/slasher.rs b/slasher/src/slasher.rs index fc8e8453c89..0bb7c9c3ffe 100644 --- a/slasher/src/slasher.rs +++ b/slasher/src/slasher.rs @@ -13,7 +13,8 @@ use slog::{debug, error, info, Logger}; use std::collections::HashSet; use std::sync::Arc; use types::{ - AttesterSlashing, Epoch, EthSpec, IndexedAttestation, ProposerSlashing, SignedBeaconBlockHeader, + AttesterSlashing, ChainSpec, Epoch, EthSpec, IndexedAttestation, ProposerSlashing, + SignedBeaconBlockHeader, }; #[derive(Debug)] @@ -28,10 +29,10 @@ pub struct Slasher { } impl Slasher { - pub fn open(config: Config, log: Logger) -> Result { + pub fn open(config: Config, spec: Arc, log: Logger) -> Result { config.validate()?; let config = Arc::new(config); - let db = SlasherDB::open(config.clone(), log.clone())?; + let db = SlasherDB::open(config.clone(), spec, log.clone())?; let attester_slashings = Mutex::new(HashSet::new()); let proposer_slashings = Mutex::new(HashSet::new()); let attestation_queue = AttestationQueue::default(); diff --git a/slasher/src/test_utils.rs b/slasher/src/test_utils.rs index 7019a8aed76..453d0e66670 100644 --- a/slasher/src/test_utils.rs +++ b/slasher/src/test_utils.rs @@ -1,9 +1,10 @@ use std::collections::HashSet; +use std::sync::Arc; use types::{ indexed_attestation::{IndexedAttestationBase, IndexedAttestationElectra}, AggregateSignature, AttestationData, AttesterSlashing, AttesterSlashingBase, - AttesterSlashingElectra, BeaconBlockHeader, Checkpoint, Epoch, Hash256, IndexedAttestation, - MainnetEthSpec, Signature, SignedBeaconBlockHeader, Slot, + AttesterSlashingElectra, BeaconBlockHeader, ChainSpec, Checkpoint, Epoch, EthSpec, Hash256, + IndexedAttestation, MainnetEthSpec, Signature, SignedBeaconBlockHeader, Slot, }; pub type E = MainnetEthSpec; @@ -145,3 +146,7 @@ pub fn block(slot: u64, proposer_index: u64, block_root: u64) -> SignedBeaconBlo signature: Signature::empty(), } } + +pub fn chain_spec() -> Arc { + Arc::new(E::default_spec()) +} diff --git a/slasher/tests/attester_slashings.rs b/slasher/tests/attester_slashings.rs index b74e491e4b3..902141d9710 100644 --- a/slasher/tests/attester_slashings.rs +++ b/slasher/tests/attester_slashings.rs @@ -6,7 +6,8 @@ use rayon::prelude::*; use slasher::{ config::DEFAULT_CHUNK_SIZE, test_utils::{ - att_slashing, indexed_att, indexed_att_electra, slashed_validators_from_slashings, E, + att_slashing, chain_spec, indexed_att, indexed_att_electra, + slashed_validators_from_slashings, E, }, Config, Slasher, }; @@ -270,7 +271,8 @@ fn slasher_test( ) { let tempdir = tempdir().unwrap(); let config = Config::new(tempdir.path().into()); - let slasher = Slasher::open(config, test_logger()).unwrap(); + let spec = chain_spec(); + let slasher = Slasher::open(config, spec, test_logger()).unwrap(); let current_epoch = Epoch::new(current_epoch); for (i, attestation) in attestations.iter().enumerate() { @@ -299,7 +301,8 @@ fn parallel_slasher_test( ) { let tempdir = tempdir().unwrap(); let config = Config::new(tempdir.path().into()); - let slasher = Slasher::open(config, test_logger()).unwrap(); + let spec = chain_spec(); + let slasher = Slasher::open(config, spec, test_logger()).unwrap(); let current_epoch = Epoch::new(current_epoch); attestations diff --git a/slasher/tests/proposer_slashings.rs b/slasher/tests/proposer_slashings.rs index 3b7b8ed583c..2d2738087dc 100644 --- a/slasher/tests/proposer_slashings.rs +++ b/slasher/tests/proposer_slashings.rs @@ -2,7 +2,7 @@ use logging::test_logger; use slasher::{ - test_utils::{block as test_block, E}, + test_utils::{block as test_block, chain_spec, E}, Config, Slasher, }; use tempfile::tempdir; @@ -12,7 +12,8 @@ use types::{Epoch, EthSpec}; fn empty_pruning() { let tempdir = tempdir().unwrap(); let config = Config::new(tempdir.path().into()); - let slasher = Slasher::::open(config, test_logger()).unwrap(); + let spec = chain_spec(); + let slasher = Slasher::::open(config, spec, test_logger()).unwrap(); slasher.prune_database(Epoch::new(0)).unwrap(); } @@ -24,8 +25,9 @@ fn block_pruning() { let mut config = Config::new(tempdir.path().into()); config.chunk_size = 2; config.history_length = 2; + let spec = chain_spec(); - let slasher = Slasher::::open(config.clone(), test_logger()).unwrap(); + let slasher = Slasher::::open(config.clone(), spec, test_logger()).unwrap(); let current_epoch = Epoch::from(2 * config.history_length); // Pruning the empty database should be safe. diff --git a/slasher/tests/random.rs b/slasher/tests/random.rs index ce0e42df1d3..ebfe0ef4e97 100644 --- a/slasher/tests/random.rs +++ b/slasher/tests/random.rs @@ -4,7 +4,7 @@ use logging::test_logger; use rand::prelude::*; use slasher::{ test_utils::{ - block, indexed_att, slashed_validators_from_attestations, + block, chain_spec, indexed_att, slashed_validators_from_attestations, slashed_validators_from_slashings, E, }, Config, Slasher, @@ -49,7 +49,9 @@ fn random_test(seed: u64, test_config: TestConfig) { config.chunk_size = 1 << chunk_size_exponent; config.history_length = 1 << rng.gen_range(chunk_size_exponent..chunk_size_exponent + 3); - let slasher = Slasher::::open(config.clone(), test_logger()).unwrap(); + let spec = chain_spec(); + + let slasher = Slasher::::open(config.clone(), spec, test_logger()).unwrap(); let validators = (0..num_validators as u64).collect::>(); diff --git a/slasher/tests/wrap_around.rs b/slasher/tests/wrap_around.rs index d2c876d3630..9a42aeb60ba 100644 --- a/slasher/tests/wrap_around.rs +++ b/slasher/tests/wrap_around.rs @@ -1,7 +1,10 @@ #![cfg(any(feature = "mdbx", feature = "lmdb"))] use logging::test_logger; -use slasher::{test_utils::indexed_att, Config, Slasher}; +use slasher::{ + test_utils::{chain_spec, indexed_att}, + Config, Slasher, +}; use tempfile::tempdir; use types::Epoch; @@ -9,11 +12,12 @@ use types::Epoch; fn attestation_pruning_empty_wrap_around() { let tempdir = tempdir().unwrap(); let mut config = Config::new(tempdir.path().into()); + let spec = chain_spec(); config.validator_chunk_size = 1; config.chunk_size = 16; config.history_length = 16; - let slasher = Slasher::open(config.clone(), test_logger()).unwrap(); + let slasher = Slasher::open(config.clone(), spec, test_logger()).unwrap(); let v = vec![0]; let history_length = config.history_length as u64; From ebbb17b6bcef883072aa1acb24dc4985a6d0b1ee Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 21 Jun 2024 14:13:22 +1000 Subject: [PATCH 173/227] Delete slasher schema v4 --- slasher/src/database.rs | 2 +- slasher/src/migrate.rs | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/slasher/src/database.rs b/slasher/src/database.rs index 2463c77f41e..81e9faa5bdd 100644 --- a/slasher/src/database.rs +++ b/slasher/src/database.rs @@ -25,7 +25,7 @@ use types::{ }; /// Current database schema version, to check compatibility of on-disk DB with software. -pub const CURRENT_SCHEMA_VERSION: u64 = 4; +pub const CURRENT_SCHEMA_VERSION: u64 = 3; /// Metadata about the slashing database itself. const METADATA_DB: &str = "metadata"; diff --git a/slasher/src/migrate.rs b/slasher/src/migrate.rs index ce298caaf2c..674ab9c132d 100644 --- a/slasher/src/migrate.rs +++ b/slasher/src/migrate.rs @@ -17,10 +17,6 @@ impl SlasherDB { software_schema_version: CURRENT_SCHEMA_VERSION, }), (x, y) if x == y => Ok(self), - (3, 4) => { - // TODO(electra): db migration due to `IndexedAttestationOnDisk` - Ok(self) - } (_, _) => Err(Error::IncompatibleSchemaVersion { database_schema_version: schema_version, software_schema_version: CURRENT_SCHEMA_VERSION, From 13b1b0596058ab37f1c1070d6ce88ab5c981ea62 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 21 Jun 2024 17:03:06 +1000 Subject: [PATCH 174/227] Fix clippy --- slasher/src/database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slasher/src/database.rs b/slasher/src/database.rs index 81e9faa5bdd..13c5d2b774c 100644 --- a/slasher/src/database.rs +++ b/slasher/src/database.rs @@ -865,7 +865,7 @@ mod test { let decoded_on_disk = IndexedAttestationOnDisk::from_ssz_bytes(&encoded).unwrap(); assert_eq!(decoded_on_disk, on_disk); - let decoded = on_disk.into_indexed_attestation(&spec).unwrap(); + let decoded = on_disk.into_indexed_attestation(spec).unwrap(); assert_eq!(decoded, fork_attestation); } From 339d1b8229d1b4c36ff9c13b9df64cdeba5254b7 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 21 Jun 2024 17:36:45 +1000 Subject: [PATCH 175/227] Fix compilation of beacon_chain tests --- beacon_node/beacon_chain/tests/block_verification.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 4c7a499697d..d9c9a3b6a74 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -2,7 +2,9 @@ use beacon_chain::block_verification_types::{AsBlock, ExecutedBlock, RpcBlock}; use beacon_chain::{ - test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType}, + test_utils::{ + test_spec, AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType, + }, AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes, ExecutionPendingBlock, }; use beacon_chain::{ @@ -1210,8 +1212,14 @@ async fn block_gossip_verification() { #[tokio::test] async fn verify_block_for_gossip_slashing_detection() { let slasher_dir = tempdir().unwrap(); + let spec = Arc::new(test_spec::()); let slasher = Arc::new( - Slasher::open(SlasherConfig::new(slasher_dir.path().into()), test_logger()).unwrap(), + Slasher::open( + SlasherConfig::new(slasher_dir.path().into()), + spec, + test_logger(), + ) + .unwrap(), ); let inner_slasher = slasher.clone(); From 70a80d5da0441c94047da2f51d48a96312e9c9a6 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 21 Jun 2024 09:48:35 +0200 Subject: [PATCH 176/227] Update database.rs --- slasher/src/database.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/slasher/src/database.rs b/slasher/src/database.rs index 13c5d2b774c..673e15d3b2f 100644 --- a/slasher/src/database.rs +++ b/slasher/src/database.rs @@ -819,7 +819,7 @@ impl SlasherDB { #[cfg(test)] mod test { use super::*; - use types::{Checkpoint, MainnetEthSpec}; + use types::{Checkpoint, MainnetEthSpec, Unsigned}; type E = MainnetEthSpec; @@ -830,6 +830,7 @@ mod test { AttestationData, AggregateSignature, ) -> IndexedAttestation, + committee_len: u64, ) { let attestation_data = AttestationData { slot: Slot::new(1000), @@ -845,7 +846,7 @@ mod test { }, }; - let attesting_indices = vec![1, 14, 160, 812737]; + let attesting_indices = (0..committee_len).collect::>(); let signature = AggregateSignature::infinity(); let fork_attestation = make_attestation( @@ -880,7 +881,11 @@ mod test { signature, }) }; - indexed_attestation_on_disk_roundtrip_test(&spec, make_attestation) + indexed_attestation_on_disk_roundtrip_test( + &spec, + make_attestation, + ::MaxValidatorsPerCommittee::to_u64(), + ) } #[test] @@ -893,6 +898,10 @@ mod test { signature, }) }; - indexed_attestation_on_disk_roundtrip_test(&spec, make_attestation) + indexed_attestation_on_disk_roundtrip_test( + &spec, + make_attestation, + ::MaxValidatorsPerSlot::to_u64(), + ) } } From 7509cf6d3b6a704a627bb430ff072f45d81cd01c Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 21 Jun 2024 09:55:23 +0200 Subject: [PATCH 177/227] Update per_block_processing.rs --- consensus/state_processing/src/per_block_processing.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 05621d188be..e7655b453a8 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -626,6 +626,7 @@ pub fn process_withdrawals>( // Update pending partial withdrawals [New in Electra:EIP7251] if let Some(partial_withdrawals_count) = partial_withdrawals_count { + // TODO(electra): Use efficient pop_front after milhouse release https://github.com/sigp/milhouse/pull/38 let new_partial_withdrawals = state .pending_partial_withdrawals()? .iter_from(partial_withdrawals_count)? From 8715589e40d8b39433cf02b7c0c6cd2612e28749 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 21 Jun 2024 14:36:02 +0200 Subject: [PATCH 178/227] Add electra lightclient types --- .../types/src/execution_payload_header.rs | 8 +- consensus/types/src/lib.rs | 8 +- consensus/types/src/light_client_bootstrap.rs | 36 ++++--- .../types/src/light_client_finality_update.rs | 99 ++++++++++--------- consensus/types/src/light_client_header.rs | 54 ++++++++-- .../src/light_client_optimistic_update.rs | 37 ++++--- consensus/types/src/light_client_update.rs | 31 +++++- 7 files changed, 181 insertions(+), 92 deletions(-) diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index a3df69652c2..b8d0c8c2b0f 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -118,12 +118,8 @@ impl ExecutionPayloadHeader { pub fn ssz_max_var_len_for_fork(fork_name: ForkName) -> usize { // Matching here in case variable fields are added in future forks. match fork_name { - ForkName::Base - | ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb - | ForkName::Electra => { + ForkName::Base | ForkName::Altair => 0, + ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra => { // Max size of variable length `extra_data` field E::max_extra_data_bytes() * ::ssz_fixed_len() } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 2a73ff0f37a..1683ae6c977 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -181,22 +181,24 @@ pub use crate::indexed_attestation::{ }; pub use crate::light_client_bootstrap::{ LightClientBootstrap, LightClientBootstrapAltair, LightClientBootstrapCapella, - LightClientBootstrapDeneb, + LightClientBootstrapDeneb, LightClientBootstrapElectra, }; pub use crate::light_client_finality_update::{ LightClientFinalityUpdate, LightClientFinalityUpdateAltair, LightClientFinalityUpdateCapella, - LightClientFinalityUpdateDeneb, + LightClientFinalityUpdateDeneb, LightClientFinalityUpdateElectra, }; pub use crate::light_client_header::{ LightClientHeader, LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, + LightClientHeaderElectra, }; pub use crate::light_client_optimistic_update::{ LightClientOptimisticUpdate, LightClientOptimisticUpdateAltair, LightClientOptimisticUpdateCapella, LightClientOptimisticUpdateDeneb, + LightClientOptimisticUpdateElectra, }; pub use crate::light_client_update::{ Error as LightClientError, LightClientUpdate, LightClientUpdateAltair, - LightClientUpdateCapella, LightClientUpdateDeneb, + LightClientUpdateCapella, LightClientUpdateDeneb, LightClientUpdateElectra, }; pub use crate::participation_flags::ParticipationFlags; pub use crate::participation_list::ParticipationList; diff --git a/consensus/types/src/light_client_bootstrap.rs b/consensus/types/src/light_client_bootstrap.rs index 61da0e1b117..e3a85744ded 100644 --- a/consensus/types/src/light_client_bootstrap.rs +++ b/consensus/types/src/light_client_bootstrap.rs @@ -1,7 +1,8 @@ use crate::{ light_client_update::*, test_utils::TestRandom, BeaconState, ChainSpec, EthSpec, FixedVector, ForkName, ForkVersionDeserialize, Hash256, LightClientHeader, LightClientHeaderAltair, - LightClientHeaderCapella, LightClientHeaderDeneb, SignedBeaconBlock, Slot, SyncCommittee, + LightClientHeaderCapella, LightClientHeaderDeneb, LightClientHeaderElectra, SignedBeaconBlock, + Slot, SyncCommittee, }; use derivative::Derivative; use serde::{Deserialize, Deserializer, Serialize}; @@ -16,7 +17,7 @@ use tree_hash_derive::TreeHash; /// A LightClientBootstrap is the initializer we send over to light_client nodes /// that are trying to generate their basic storage when booting up. #[superstruct( - variants(Altair, Capella, Deneb), + variants(Altair, Capella, Deneb, Electra), variant_attributes( derive( Debug, @@ -51,6 +52,8 @@ pub struct LightClientBootstrap { pub header: LightClientHeaderCapella, #[superstruct(only(Deneb), partial_getter(rename = "header_deneb"))] pub header: LightClientHeaderDeneb, + #[superstruct(only(Electra), partial_getter(rename = "header_electra"))] + pub header: LightClientHeaderElectra, /// The `SyncCommittee` used in the requested period. pub current_sync_committee: Arc>, /// Merkle proof for sync committee @@ -66,6 +69,7 @@ impl LightClientBootstrap { Self::Altair(_) => func(ForkName::Altair), Self::Capella(_) => func(ForkName::Capella), Self::Deneb(_) => func(ForkName::Deneb), + Self::Electra(_) => func(ForkName::Electra), } } @@ -82,9 +86,8 @@ impl LightClientBootstrap { Self::Altair(LightClientBootstrapAltair::from_ssz_bytes(bytes)?) } ForkName::Capella => Self::Capella(LightClientBootstrapCapella::from_ssz_bytes(bytes)?), - ForkName::Deneb | ForkName::Electra => { - Self::Deneb(LightClientBootstrapDeneb::from_ssz_bytes(bytes)?) - } + ForkName::Deneb => Self::Deneb(LightClientBootstrapDeneb::from_ssz_bytes(bytes)?), + ForkName::Electra => Self::Electra(LightClientBootstrapElectra::from_ssz_bytes(bytes)?), ForkName::Base => { return Err(ssz::DecodeError::BytesInvalid(format!( "LightClientBootstrap decoding for {fork_name} not implemented" @@ -97,18 +100,16 @@ impl LightClientBootstrap { #[allow(clippy::arithmetic_side_effects)] pub fn ssz_max_len_for_fork(fork_name: ForkName) -> usize { - // TODO(electra): review electra changes - match fork_name { + let fixed_len = match fork_name { ForkName::Base => 0, - ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb - | ForkName::Electra => { + ForkName::Altair | ForkName::Bellatrix => { as Encode>::ssz_fixed_len() - + LightClientHeader::::ssz_max_var_len_for_fork(fork_name) } - } + ForkName::Capella => as Encode>::ssz_fixed_len(), + ForkName::Deneb => as Encode>::ssz_fixed_len(), + ForkName::Electra => as Encode>::ssz_fixed_len(), + }; + fixed_len + LightClientHeader::::ssz_max_var_len_for_fork(fork_name) } pub fn from_beacon_state( @@ -138,11 +139,16 @@ impl LightClientBootstrap { current_sync_committee, current_sync_committee_branch, }), - ForkName::Deneb | ForkName::Electra => Self::Deneb(LightClientBootstrapDeneb { + ForkName::Deneb => Self::Deneb(LightClientBootstrapDeneb { header: LightClientHeaderDeneb::block_to_light_client_header(block)?, current_sync_committee, current_sync_committee_branch, }), + ForkName::Electra => Self::Electra(LightClientBootstrapElectra { + header: LightClientHeaderElectra::block_to_light_client_header(block)?, + current_sync_committee, + current_sync_committee_branch, + }), }; Ok(light_client_bootstrap) diff --git a/consensus/types/src/light_client_finality_update.rs b/consensus/types/src/light_client_finality_update.rs index 29c526e2916..a9e24e03db1 100644 --- a/consensus/types/src/light_client_finality_update.rs +++ b/consensus/types/src/light_client_finality_update.rs @@ -2,7 +2,8 @@ use super::{EthSpec, FixedVector, Hash256, LightClientHeader, Slot, SyncAggregat use crate::ChainSpec; use crate::{ light_client_update::*, test_utils::TestRandom, ForkName, ForkVersionDeserialize, - LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, SignedBeaconBlock, + LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, + LightClientHeaderElectra, SignedBeaconBlock, }; use derivative::Derivative; use serde::{Deserialize, Deserializer, Serialize}; @@ -15,7 +16,7 @@ use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[superstruct( - variants(Altair, Capella, Deneb), + variants(Altair, Capella, Deneb, Electra), variant_attributes( derive( Debug, @@ -50,6 +51,8 @@ pub struct LightClientFinalityUpdate { pub attested_header: LightClientHeaderCapella, #[superstruct(only(Deneb), partial_getter(rename = "attested_header_deneb"))] pub attested_header: LightClientHeaderDeneb, + #[superstruct(only(Electra), partial_getter(rename = "attested_header_electra"))] + pub attested_header: LightClientHeaderElectra, /// The last `BeaconBlockHeader` from the last attested finalized block (end of epoch). #[superstruct(only(Altair), partial_getter(rename = "finalized_header_altair"))] pub finalized_header: LightClientHeaderAltair, @@ -57,6 +60,8 @@ pub struct LightClientFinalityUpdate { pub finalized_header: LightClientHeaderCapella, #[superstruct(only(Deneb), partial_getter(rename = "finalized_header_deneb"))] pub finalized_header: LightClientHeaderDeneb, + #[superstruct(only(Electra), partial_getter(rename = "finalized_header_electra"))] + pub finalized_header: LightClientHeaderElectra, /// Merkle proof attesting finalized header. #[test_random(default)] pub finality_branch: FixedVector, @@ -80,7 +85,7 @@ impl LightClientFinalityUpdate { .map_err(|_| Error::InconsistentFork)? { ForkName::Altair | ForkName::Bellatrix => { - let finality_update = LightClientFinalityUpdateAltair { + Self::Altair(LightClientFinalityUpdateAltair { attested_header: LightClientHeaderAltair::block_to_light_client_header( attested_block, )?, @@ -90,37 +95,42 @@ impl LightClientFinalityUpdate { finality_branch, sync_aggregate, signature_slot, - }; - Self::Altair(finality_update) - } - ForkName::Capella => { - let finality_update = LightClientFinalityUpdateCapella { - attested_header: LightClientHeaderCapella::block_to_light_client_header( - attested_block, - )?, - finalized_header: LightClientHeaderCapella::block_to_light_client_header( - finalized_block, - )?, - finality_branch, - sync_aggregate, - signature_slot, - }; - Self::Capella(finality_update) - } - ForkName::Deneb | ForkName::Electra => { - let finality_update = LightClientFinalityUpdateDeneb { - attested_header: LightClientHeaderDeneb::block_to_light_client_header( - attested_block, - )?, - finalized_header: LightClientHeaderDeneb::block_to_light_client_header( - finalized_block, - )?, - finality_branch, - sync_aggregate, - signature_slot, - }; - Self::Deneb(finality_update) + }) } + ForkName::Capella => Self::Capella(LightClientFinalityUpdateCapella { + attested_header: LightClientHeaderCapella::block_to_light_client_header( + attested_block, + )?, + finalized_header: LightClientHeaderCapella::block_to_light_client_header( + finalized_block, + )?, + finality_branch, + sync_aggregate, + signature_slot, + }), + ForkName::Deneb => Self::Deneb(LightClientFinalityUpdateDeneb { + attested_header: LightClientHeaderDeneb::block_to_light_client_header( + attested_block, + )?, + finalized_header: LightClientHeaderDeneb::block_to_light_client_header( + finalized_block, + )?, + finality_branch, + sync_aggregate, + signature_slot, + }), + ForkName::Electra => Self::Electra(LightClientFinalityUpdateElectra { + attested_header: LightClientHeaderElectra::block_to_light_client_header( + attested_block, + )?, + finalized_header: LightClientHeaderElectra::block_to_light_client_header( + finalized_block, + )?, + finality_branch, + sync_aggregate, + signature_slot, + }), + ForkName::Base => return Err(Error::AltairForkNotActive), }; @@ -135,6 +145,7 @@ impl LightClientFinalityUpdate { Self::Altair(_) => func(ForkName::Altair), Self::Capella(_) => func(ForkName::Capella), Self::Deneb(_) => func(ForkName::Deneb), + Self::Electra(_) => func(ForkName::Electra), } } @@ -153,8 +164,9 @@ impl LightClientFinalityUpdate { ForkName::Capella => { Self::Capella(LightClientFinalityUpdateCapella::from_ssz_bytes(bytes)?) } - ForkName::Deneb | ForkName::Electra => { - Self::Deneb(LightClientFinalityUpdateDeneb::from_ssz_bytes(bytes)?) + ForkName::Deneb => Self::Deneb(LightClientFinalityUpdateDeneb::from_ssz_bytes(bytes)?), + ForkName::Electra => { + Self::Electra(LightClientFinalityUpdateElectra::from_ssz_bytes(bytes)?) } ForkName::Base => { return Err(ssz::DecodeError::BytesInvalid(format!( @@ -168,18 +180,17 @@ impl LightClientFinalityUpdate { #[allow(clippy::arithmetic_side_effects)] pub fn ssz_max_len_for_fork(fork_name: ForkName) -> usize { - // TODO(electra): review electra changes - match fork_name { + let fixed_size = match fork_name { ForkName::Base => 0, - ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb - | ForkName::Electra => { + ForkName::Altair | ForkName::Bellatrix => { as Encode>::ssz_fixed_len() - + 2 * LightClientHeader::::ssz_max_var_len_for_fork(fork_name) } - } + ForkName::Capella => as Encode>::ssz_fixed_len(), + ForkName::Deneb => as Encode>::ssz_fixed_len(), + ForkName::Electra => as Encode>::ssz_fixed_len(), + }; + // `2 *` because there are two headers in the update + fixed_size + 2 * LightClientHeader::::ssz_max_var_len_for_fork(fork_name) } } diff --git a/consensus/types/src/light_client_header.rs b/consensus/types/src/light_client_header.rs index 213ec90f955..1d6432ed6f3 100644 --- a/consensus/types/src/light_client_header.rs +++ b/consensus/types/src/light_client_header.rs @@ -4,7 +4,7 @@ use crate::ForkVersionDeserialize; use crate::{light_client_update::*, BeaconBlockBody}; use crate::{ test_utils::TestRandom, EthSpec, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, - FixedVector, Hash256, SignedBeaconBlock, + ExecutionPayloadHeaderElectra, FixedVector, Hash256, SignedBeaconBlock, }; use crate::{BeaconBlockHeader, ExecutionPayloadHeader}; use derivative::Derivative; @@ -17,7 +17,7 @@ use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[superstruct( - variants(Altair, Capella, Deneb), + variants(Altair, Capella, Deneb, Electra), variant_attributes( derive( Debug, @@ -54,8 +54,13 @@ pub struct LightClientHeader { pub execution: ExecutionPayloadHeaderCapella, #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_header_deneb"))] pub execution: ExecutionPayloadHeaderDeneb, + #[superstruct( + only(Electra), + partial_getter(rename = "execution_payload_header_electra") + )] + pub execution: ExecutionPayloadHeaderElectra, - #[superstruct(only(Capella, Deneb))] + #[superstruct(only(Capella, Deneb, Electra))] pub execution_branch: FixedVector, #[ssz(skip_serializing, skip_deserializing)] @@ -81,9 +86,12 @@ impl LightClientHeader { ForkName::Capella => LightClientHeader::Capella( LightClientHeaderCapella::block_to_light_client_header(block)?, ), - ForkName::Deneb | ForkName::Electra => LightClientHeader::Deneb( + ForkName::Deneb => LightClientHeader::Deneb( LightClientHeaderDeneb::block_to_light_client_header(block)?, ), + ForkName::Electra => LightClientHeader::Electra( + LightClientHeaderElectra::block_to_light_client_header(block)?, + ), }; Ok(header) } @@ -96,9 +104,12 @@ impl LightClientHeader { ForkName::Capella => { LightClientHeader::Capella(LightClientHeaderCapella::from_ssz_bytes(bytes)?) } - ForkName::Deneb | ForkName::Electra => { + ForkName::Deneb => { LightClientHeader::Deneb(LightClientHeaderDeneb::from_ssz_bytes(bytes)?) } + ForkName::Electra => { + LightClientHeader::Electra(LightClientHeaderElectra::from_ssz_bytes(bytes)?) + } ForkName::Base => { return Err(ssz::DecodeError::BytesInvalid(format!( "LightClientHeader decoding for {fork_name} not implemented" @@ -192,6 +203,34 @@ impl LightClientHeaderDeneb { } } +impl LightClientHeaderElectra { + pub fn block_to_light_client_header(block: &SignedBeaconBlock) -> Result { + let payload = block + .message() + .execution_payload()? + .execution_payload_electra()?; + + let header = ExecutionPayloadHeaderElectra::from(payload); + let beacon_block_body = BeaconBlockBody::from( + block + .message() + .body_electra() + .map_err(|_| Error::BeaconBlockBodyError)? + .to_owned(), + ); + + let execution_branch = + beacon_block_body.block_body_merkle_proof(EXECUTION_PAYLOAD_INDEX)?; + + Ok(LightClientHeaderElectra { + beacon: block.message().block_header(), + execution: header, + execution_branch: FixedVector::new(execution_branch)?, + _phantom_data: PhantomData, + }) + } +} + impl ForkVersionDeserialize for LightClientHeader { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, @@ -204,9 +243,12 @@ impl ForkVersionDeserialize for LightClientHeader { ForkName::Capella => serde_json::from_value(value) .map(|light_client_header| Self::Capella(light_client_header)) .map_err(serde::de::Error::custom), - ForkName::Deneb | ForkName::Electra => serde_json::from_value(value) + ForkName::Deneb => serde_json::from_value(value) .map(|light_client_header| Self::Deneb(light_client_header)) .map_err(serde::de::Error::custom), + ForkName::Electra => serde_json::from_value(value) + .map(|light_client_header| Self::Electra(light_client_header)) + .map_err(serde::de::Error::custom), ForkName::Base => Err(serde::de::Error::custom(format!( "LightClientHeader deserialization for {fork_name} not implemented" ))), diff --git a/consensus/types/src/light_client_optimistic_update.rs b/consensus/types/src/light_client_optimistic_update.rs index 4727673f6c0..708f24e7701 100644 --- a/consensus/types/src/light_client_optimistic_update.rs +++ b/consensus/types/src/light_client_optimistic_update.rs @@ -2,7 +2,7 @@ use super::{EthSpec, ForkName, ForkVersionDeserialize, LightClientHeader, Slot, use crate::test_utils::TestRandom; use crate::{ light_client_update::*, ChainSpec, LightClientHeaderAltair, LightClientHeaderCapella, - LightClientHeaderDeneb, SignedBeaconBlock, + LightClientHeaderDeneb, LightClientHeaderElectra, SignedBeaconBlock, }; use derivative::Derivative; use serde::{Deserialize, Deserializer, Serialize}; @@ -18,7 +18,7 @@ use tree_hash_derive::TreeHash; /// A LightClientOptimisticUpdate is the update we send on each slot, /// it is based off the current unfinalized epoch is verified only against BLS signature. #[superstruct( - variants(Altair, Capella, Deneb), + variants(Altair, Capella, Deneb, Electra), variant_attributes( derive( Debug, @@ -53,6 +53,8 @@ pub struct LightClientOptimisticUpdate { pub attested_header: LightClientHeaderCapella, #[superstruct(only(Deneb), partial_getter(rename = "attested_header_deneb"))] pub attested_header: LightClientHeaderDeneb, + #[superstruct(only(Electra), partial_getter(rename = "attested_header_electra"))] + pub attested_header: LightClientHeaderElectra, /// current sync aggregate pub sync_aggregate: SyncAggregate, /// Slot of the sync aggregated signature @@ -86,13 +88,20 @@ impl LightClientOptimisticUpdate { sync_aggregate, signature_slot, }), - ForkName::Deneb | ForkName::Electra => Self::Deneb(LightClientOptimisticUpdateDeneb { + ForkName::Deneb => Self::Deneb(LightClientOptimisticUpdateDeneb { attested_header: LightClientHeaderDeneb::block_to_light_client_header( attested_block, )?, sync_aggregate, signature_slot, }), + ForkName::Electra => Self::Electra(LightClientOptimisticUpdateElectra { + attested_header: LightClientHeaderElectra::block_to_light_client_header( + attested_block, + )?, + sync_aggregate, + signature_slot, + }), ForkName::Base => return Err(Error::AltairForkNotActive), }; @@ -107,6 +116,7 @@ impl LightClientOptimisticUpdate { Self::Altair(_) => func(ForkName::Altair), Self::Capella(_) => func(ForkName::Capella), Self::Deneb(_) => func(ForkName::Deneb), + Self::Electra(_) => func(ForkName::Electra), } } @@ -139,9 +149,12 @@ impl LightClientOptimisticUpdate { ForkName::Capella => { Self::Capella(LightClientOptimisticUpdateCapella::from_ssz_bytes(bytes)?) } - ForkName::Deneb | ForkName::Electra => { + ForkName::Deneb => { Self::Deneb(LightClientOptimisticUpdateDeneb::from_ssz_bytes(bytes)?) } + ForkName::Electra => { + Self::Electra(LightClientOptimisticUpdateElectra::from_ssz_bytes(bytes)?) + } ForkName::Base => { return Err(ssz::DecodeError::BytesInvalid(format!( "LightClientOptimisticUpdate decoding for {fork_name} not implemented" @@ -154,18 +167,16 @@ impl LightClientOptimisticUpdate { #[allow(clippy::arithmetic_side_effects)] pub fn ssz_max_len_for_fork(fork_name: ForkName) -> usize { - // TODO(electra): review electra changes - match fork_name { + let fixed_len = match fork_name { ForkName::Base => 0, - ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb - | ForkName::Electra => { + ForkName::Altair | ForkName::Bellatrix => { as Encode>::ssz_fixed_len() - + LightClientHeader::::ssz_max_var_len_for_fork(fork_name) } - } + ForkName::Capella => as Encode>::ssz_fixed_len(), + ForkName::Deneb => as Encode>::ssz_fixed_len(), + ForkName::Electra => as Encode>::ssz_fixed_len(), + }; + fixed_len + LightClientHeader::::ssz_max_var_len_for_fork(fork_name) } } diff --git a/consensus/types/src/light_client_update.rs b/consensus/types/src/light_client_update.rs index 002fbea2d37..76ad5689888 100644 --- a/consensus/types/src/light_client_update.rs +++ b/consensus/types/src/light_client_update.rs @@ -1,4 +1,5 @@ use super::{EthSpec, FixedVector, Hash256, Slot, SyncAggregate, SyncCommittee}; +use crate::light_client_header::LightClientHeaderElectra; use crate::{ beacon_state, test_utils::TestRandom, BeaconBlock, BeaconBlockHeader, BeaconState, ChainSpec, ForkName, ForkVersionDeserialize, LightClientHeaderAltair, LightClientHeaderCapella, @@ -76,7 +77,7 @@ impl From for Error { /// or to sync up to the last committee period, we need to have one ready for each ALTAIR period /// we go over, note: there is no need to keep all of the updates from [ALTAIR_PERIOD, CURRENT_PERIOD]. #[superstruct( - variants(Altair, Capella, Deneb), + variants(Altair, Capella, Deneb, Electra), variant_attributes( derive( Debug, @@ -111,6 +112,8 @@ pub struct LightClientUpdate { pub attested_header: LightClientHeaderCapella, #[superstruct(only(Deneb), partial_getter(rename = "attested_header_deneb"))] pub attested_header: LightClientHeaderDeneb, + #[superstruct(only(Electra), partial_getter(rename = "attested_header_electra"))] + pub attested_header: LightClientHeaderElectra, /// The `SyncCommittee` used in the next period. pub next_sync_committee: Arc>, /// Merkle proof for next sync committee @@ -122,6 +125,8 @@ pub struct LightClientUpdate { pub finalized_header: LightClientHeaderCapella, #[superstruct(only(Deneb), partial_getter(rename = "finalized_header_deneb"))] pub finalized_header: LightClientHeaderDeneb, + #[superstruct(only(Electra), partial_getter(rename = "finalized_header_electra"))] + pub finalized_header: LightClientHeaderElectra, /// Merkle proof attesting finalized header. pub finality_branch: FixedVector, /// current sync aggreggate @@ -221,7 +226,7 @@ impl LightClientUpdate { signature_slot: block.slot(), }) } - ForkName::Deneb | ForkName::Electra => { + ForkName::Deneb => { let attested_header = LightClientHeaderDeneb::block_to_light_client_header(attested_block)?; let finalized_header = @@ -236,6 +241,23 @@ impl LightClientUpdate { signature_slot: block.slot(), }) } + ForkName::Electra => { + let attested_header = + LightClientHeaderElectra::block_to_light_client_header(attested_block)?; + let finalized_header = + LightClientHeaderElectra::block_to_light_client_header(finalized_block)?; + Self::Electra(LightClientUpdateElectra { + attested_header, + next_sync_committee: attested_state.next_sync_committee()?.clone(), + next_sync_committee_branch: FixedVector::new(next_sync_committee_branch)?, + finalized_header, + finality_branch: FixedVector::new(finality_branch)?, + sync_aggregate: sync_aggregate.clone(), + signature_slot: block.slot(), + }) + } // To add a new fork, just append the new fork variant on the latest fork. Forks that + // have a distinct execution header will need a new LightClientUdpate variant only + // if you need to test or support lightclient usages }; Ok(light_client_update) @@ -247,9 +269,8 @@ impl LightClientUpdate { Self::Altair(LightClientUpdateAltair::from_ssz_bytes(bytes)?) } ForkName::Capella => Self::Capella(LightClientUpdateCapella::from_ssz_bytes(bytes)?), - ForkName::Deneb | ForkName::Electra => { - Self::Deneb(LightClientUpdateDeneb::from_ssz_bytes(bytes)?) - } + ForkName::Deneb => Self::Deneb(LightClientUpdateDeneb::from_ssz_bytes(bytes)?), + ForkName::Electra => Self::Electra(LightClientUpdateElectra::from_ssz_bytes(bytes)?), ForkName::Base => { return Err(ssz::DecodeError::BytesInvalid(format!( "LightClientUpdate decoding for {fork_name} not implemented" From 09141ec51ad32cbd823b7406b5430d76acddc2fd Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 21 Jun 2024 10:48:43 -0400 Subject: [PATCH 179/227] Update slasher/src/database.rs --- slasher/src/database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slasher/src/database.rs b/slasher/src/database.rs index 673e15d3b2f..a81a3b0580d 100644 --- a/slasher/src/database.rs +++ b/slasher/src/database.rs @@ -258,7 +258,7 @@ impl IndexedAttestationOnDisk { spec: &ChainSpec, ) -> Result, Error> { let fork_at_target_epoch = spec.fork_name_at_epoch(self.data.target.epoch); - if fork_at_target_epoch >= ForkName::Electra { + if fork_at_target_epoch.electra_enabled() { let attesting_indices = VariableList::new(self.attesting_indices)?; Ok(IndexedAttestation::Electra(IndexedAttestationElectra { attesting_indices, From 8fc533368c6f94e3ff56df795f7db18444215a69 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 21 Jun 2024 10:57:35 -0400 Subject: [PATCH 180/227] fix imports --- slasher/src/database.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/slasher/src/database.rs b/slasher/src/database.rs index a81a3b0580d..801abe9283b 100644 --- a/slasher/src/database.rs +++ b/slasher/src/database.rs @@ -19,9 +19,9 @@ use std::marker::PhantomData; use std::sync::Arc; use tree_hash::TreeHash; use types::{ - AggregateSignature, AttestationData, ChainSpec, Epoch, EthSpec, ForkName, Hash256, - IndexedAttestation, IndexedAttestationBase, IndexedAttestationElectra, ProposerSlashing, - SignedBeaconBlockHeader, Slot, VariableList, + AggregateSignature, AttestationData, ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, + IndexedAttestationBase, IndexedAttestationElectra, ProposerSlashing, SignedBeaconBlockHeader, + Slot, VariableList, }; /// Current database schema version, to check compatibility of on-disk DB with software. @@ -819,7 +819,7 @@ impl SlasherDB { #[cfg(test)] mod test { use super::*; - use types::{Checkpoint, MainnetEthSpec, Unsigned}; + use types::{Checkpoint, ForkName, MainnetEthSpec, Unsigned}; type E = MainnetEthSpec; From 68fd7a7881d84d8ac536d8542b8be55a65e9fb8d Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 21 Jun 2024 13:09:56 -0400 Subject: [PATCH 181/227] Update beacon_node/beacon_chain/src/attestation_verification.rs --- beacon_node/beacon_chain/src/attestation_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index c7f7bb58dab..45db6f23a35 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -1396,7 +1396,7 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( /// `beacon_chain.shuffling_cache` or it might involve reading it from a state from the DB. Due to /// the complexities of `RwLock`s on the shuffling cache, a simple `Cow` isn't suitable here. /// -/// If the committees for an `attestation`'s slot isn't found in the `shuffling_cache`, we will read a state +/// If the committees for an `attestation`'s slot aren't found in the `shuffling_cache`, we will read a state /// from disk and then update the `shuffling_cache`. /// /// Committees are sorted by ascending index order 0..committees_per_slot From d137881614d94324194a26b718623714eef66cb9 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 21 Jun 2024 13:13:07 -0400 Subject: [PATCH 182/227] Update beacon_node/beacon_chain/src/attestation_verification.rs --- beacon_node/beacon_chain/src/attestation_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 45db6f23a35..06fba937d84 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -1392,7 +1392,7 @@ pub fn obtain_indexed_attestation_and_committees_per_slot( /// Runs the `map_fn` with the committee and committee count per slot for the given `attestation`. /// /// This function exists in this odd "map" pattern because efficiently obtaining the committees for -/// an attestations slot can be complex. It might involve reading straight from the +/// an attestation's slot can be complex. It might involve reading straight from the /// `beacon_chain.shuffling_cache` or it might involve reading it from a state from the DB. Due to /// the complexities of `RwLock`s on the shuffling cache, a simple `Cow` isn't suitable here. /// From 7d243f9f19a871738778473802f39d4173e5f41c Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 25 Jun 2024 13:55:08 -0700 Subject: [PATCH 183/227] clean up enabled and disabled fork logic in ef tests --- consensus/types/src/fork_name.rs | 12 ++++++++++++ .../ef_tests/src/cases/epoch_processing.rs | 19 +++++++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index 96f206d4543..4d6802d1ab7 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -120,6 +120,18 @@ impl ForkName { } } + pub fn altair_enabled(self) -> bool { + self >= ForkName::Altair + } + + pub fn bellatrix_enabled(self) -> bool { + self >= ForkName::Bellatrix + } + + pub fn capella_enabled(self) -> bool { + self >= ForkName::Capella + } + pub fn deneb_enabled(self) -> bool { self >= ForkName::Deneb } diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index 390dab9a2b1..dfd782a22b3 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -342,28 +342,27 @@ impl> Case for EpochProcessing { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - if fork_name > ForkName::Base && (T::name() == "participation_record_updates") { + if !fork_name.altair_enabled() + && (T::name() == "sync_committee_updates" + || T::name() == "inactivity_updates" + || T::name() == "participation_flag_updates") + { return false; } - if fork_name > ForkName::Bellatrix && T::name() == "historical_roots_update" { + if fork_name.altair_enabled() && T::name() == "participation_record_updates" { return false; } - if fork_name < ForkName::Capella && T::name() == "historical_summaries_update" { + if !fork_name.capella_enabled() && T::name() == "historical_summaries_update" { return false; } - if fork_name < ForkName::Altair - && (T::name() == "sync_committee_updates" - || T::name() == "inactivity_updates" - || T::name() == "participation_flag_updates" - || T::name() == "historical_summaries_update") - { + if fork_name.capella_enabled() && T::name() == "historical_roots_update" { return false; } - if fork_name < ForkName::Electra + if !fork_name.electra_enabled() && (T::name() == "pending_consolidations" || T::name() == "pending_balance_deposits") { return false; From f48b704b2b21038b12d21e7c5cfbcb47bcaeb90a Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Wed, 26 Jun 2024 10:17:44 -0500 Subject: [PATCH 184/227] Electra devnet 1 process operations (#5994) * Implement `process_consolidation_requests()` * Finish Changes to `process_operations()` * Fix Lint * Fix test * use electra_enabled() --- beacon_node/execution_layer/src/engine_api.rs | 2 +- .../src/engine_api/json_structures.rs | 8 +- beacon_node/execution_layer/src/lib.rs | 2 +- beacon_node/http_api/tests/tests.rs | 2 +- .../src/per_block_processing/errors.rs | 40 ------ .../process_operations.rs | 131 ++++++++++++++---- testing/ef_tests/src/cases/operations.rs | 7 +- 7 files changed, 113 insertions(+), 79 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 6e313fa7ad4..2391d8c087b 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -20,7 +20,7 @@ use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use strum::IntoStaticStr; use superstruct::superstruct; -use types::execution_payload::{ConsolidationRequests, DepositRequests, WithdrawalRequests}; +use types::execution_payload::{DepositRequests, WithdrawalRequests}; pub use types::{ Address, BeaconBlockRef, ConsolidationRequest, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 0113bcbb347..b5f04fed19e 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -223,11 +223,7 @@ impl From> for JsonExecutionPayloadV4 .map(Into::into) .collect::>() .into(), - consolidation_requests: payload - .consolidation_requests - .into_iter() - .collect::>() - .into(), + consolidation_requests: payload.consolidation_requests, } } } @@ -356,7 +352,7 @@ impl From> for ExecutionPayloadElectra .map(Into::into) .collect::>() .into(), - consolidation_requests: payload.consolidation_requests.into(), + consolidation_requests: payload.consolidation_requests, } } } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index ff7ada4debb..cd41fc97a10 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -2034,7 +2034,7 @@ impl ExecutionLayer { excess_blob_gas: electra_block.excess_blob_gas, deposit_requests, withdrawal_requests, - consolidation_requests: consolidation_requests, + consolidation_requests, }) } }; diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 5baf96091bf..1892e2e06ce 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -3210,7 +3210,7 @@ impl ApiTester { .get_validator_aggregate_attestation_v2( attestation.data().slot, attestation.data().tree_hash_root(), - attestation.committee_index(), + attestation.committee_index().expect("committee index"), ) .await .unwrap() diff --git a/consensus/state_processing/src/per_block_processing/errors.rs b/consensus/state_processing/src/per_block_processing/errors.rs index cebb10b6071..fdeec6f08c3 100644 --- a/consensus/state_processing/src/per_block_processing/errors.rs +++ b/consensus/state_processing/src/per_block_processing/errors.rs @@ -89,46 +89,6 @@ pub enum BlockProcessingError { found: Hash256, }, WithdrawalCredentialsInvalid, - TooManyPendingConsolidations { - consolidations: usize, - limit: usize, - }, - ConsolidationChurnLimitTooLow { - churn_limit: u64, - minimum: u64, - }, - MatchingSourceTargetConsolidation { - index: u64, - }, - InactiveConsolidationSource { - index: u64, - current_epoch: Epoch, - }, - InactiveConsolidationTarget { - index: u64, - current_epoch: Epoch, - }, - SourceValidatorExiting { - index: u64, - }, - TargetValidatorExiting { - index: u64, - }, - FutureConsolidationEpoch { - current_epoch: Epoch, - consolidation_epoch: Epoch, - }, - NoSourceExecutionWithdrawalCredential { - index: u64, - }, - NoTargetExecutionWithdrawalCredential { - index: u64, - }, - MismatchedWithdrawalCredentials { - source_address: Address, - target_address: Address, - }, - InavlidConsolidationSignature, PendingAttestationInElectra, } diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index ca9eae148a3..be5b60fe0b4 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -38,17 +38,15 @@ pub fn process_operations>( process_bls_to_execution_changes(state, bls_to_execution_changes, verify_signatures, spec)?; } - if state.fork_name_unchecked() >= ForkName::Electra { - let requests = block_body.execution_payload()?.withdrawal_requests()?; - if let Some(requests) = requests { - process_execution_layer_withdrawal_requests(state, &requests, spec)?; + if state.fork_name_unchecked().electra_enabled() { + state.update_pubkey_cache()?; + if let Some(deposit_requests) = block_body.execution_payload()?.deposit_requests()? { + process_deposit_requests(state, &deposit_requests, spec)?; } - let receipts = block_body.execution_payload()?.deposit_requests()?; - if let Some(receipts) = receipts { - process_deposit_receipts(state, &receipts, spec)?; + if let Some(withdrawal_requests) = block_body.execution_payload()?.withdrawal_requests()? { + process_withdrawal_requests(state, &withdrawal_requests, spec)?; } - let consolidations = block_body.execution_payload()?.consolidation_requests()?; - if let Some(consolidations) = consolidations { + if let Some(consolidations) = block_body.execution_payload()?.consolidation_requests()? { process_consolidation_requests(state, &consolidations, spec)?; } } @@ -373,10 +371,11 @@ pub fn process_deposits( ) -> Result<(), BlockProcessingError> { // [Modified in Electra:EIP6110] // Disable former deposit mechanism once all prior deposits are processed - // - // If `deposit_requests_start_index` does not exist as a field on `state`, electra is disabled - // which means we always want to use the old check, so this field defaults to `u64::MAX`. - let eth1_deposit_index_limit = state.deposit_requests_start_index().unwrap_or(u64::MAX); + let deposit_requests_start_index = state.deposit_requests_start_index().unwrap_or(u64::MAX); + let eth1_deposit_index_limit = std::cmp::min( + deposit_requests_start_index, + state.eth1_data().deposit_count, + ); if state.eth1_deposit_index() < eth1_deposit_index_limit { let expected_deposit_len = std::cmp::min( @@ -530,7 +529,8 @@ pub fn apply_deposit( Ok(()) } -pub fn process_execution_layer_withdrawal_requests( +// Make sure to build the pubkey cache before calling this function +pub fn process_withdrawal_requests( state: &mut BeaconState, requests: &[WithdrawalRequest], spec: &ChainSpec, @@ -547,13 +547,11 @@ pub fn process_execution_layer_withdrawal_requests( } // Verify pubkey exists - let index_opt = state.get_validator_index(&request.validator_pubkey)?; - let Some(index) = index_opt else { + let Some(index) = state.pubkey_cache().get(&request.validator_pubkey) else { continue; }; let validator = state.get_validator(index)?; - // Verify withdrawal credentials let has_correct_credential = validator.has_execution_withdrawal_credential(spec); let is_correct_source_address = validator @@ -627,21 +625,21 @@ pub fn process_execution_layer_withdrawal_requests( Ok(()) } -pub fn process_deposit_receipts( +pub fn process_deposit_requests( state: &mut BeaconState, - receipts: &[DepositRequest], + deposit_requests: &[DepositRequest], spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { - for receipt in receipts { + for request in deposit_requests { // Set deposit receipt start index if state.deposit_requests_start_index()? == spec.unset_deposit_requests_start_index { - *state.deposit_requests_start_index_mut()? = receipt.index + *state.deposit_requests_start_index_mut()? = request.index } let deposit_data = DepositData { - pubkey: receipt.pubkey, - withdrawal_credentials: receipt.withdrawal_credentials, - amount: receipt.amount, - signature: receipt.signature.clone().into(), + pubkey: request.pubkey, + withdrawal_credentials: request.withdrawal_credentials, + amount: request.amount, + signature: request.signature.clone().into(), }; apply_deposit(state, deposit_data, None, false, spec)? } @@ -649,10 +647,91 @@ pub fn process_deposit_receipts( Ok(()) } +// Make sure to build the pubkey cache before calling this function pub fn process_consolidation_requests( state: &mut BeaconState, consolidation_requests: &[ConsolidationRequest], spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { - todo!("implement process_consolidation_requests"); + for request in consolidation_requests { + process_consolidation_request(state, request, spec)?; + } + + Ok(()) +} + +pub fn process_consolidation_request( + state: &mut BeaconState, + consolidation_request: &ConsolidationRequest, + spec: &ChainSpec, +) -> Result<(), BlockProcessingError> { + // If the pending consolidations queue is full, consolidation requests are ignored + if state.pending_consolidations()?.len() == E::PendingConsolidationsLimit::to_usize() { + return Ok(()); + } + // If there is too little available consolidation churn limit, consolidation requests are ignored + if state.get_consolidation_churn_limit(spec)? <= spec.min_activation_balance { + return Ok(()); + } + + let Some(source_index) = state + .pubkey_cache() + .get(&consolidation_request.source_pubkey) + else { + // source validator doesn't exist + return Ok(()); + }; + let Some(target_index) = state + .pubkey_cache() + .get(&consolidation_request.target_pubkey) + else { + // target validator doesn't exist + return Ok(()); + }; + // Verify that source != target, so a consolidation cannot be used as an exit. + if source_index == target_index { + return Ok(()); + } + + let source_validator = state.get_validator(source_index)?; + // Verify the source withdrawal credentials + if let Some(withdrawal_address) = source_validator.get_execution_withdrawal_address(spec) { + if withdrawal_address != consolidation_request.source_address { + return Ok(()); + } + } else { + // Source doen't have execution withdrawal credentials + return Ok(()); + } + + let target_validator = state.get_validator(target_index)?; + // Verify the target has execution withdrawal credentials + if !target_validator.has_execution_withdrawal_credential(spec) { + return Ok(()); + } + + // Verify the source and target are active + let current_epoch = state.current_epoch(); + if !source_validator.is_active_at(current_epoch) + || !target_validator.is_active_at(current_epoch) + { + return Ok(()); + } + // Verify exits for source and target have not been initiated + if source_validator.exit_epoch != spec.far_future_epoch + || target_validator.exit_epoch != spec.far_future_epoch + { + return Ok(()); + } + + // Initiate source validator exit and append pending consolidation + initiate_validator_exit(state, source_index, spec)?; + state + .pending_consolidations_mut()? + .push(PendingConsolidation { + source_index: source_index as u64, + target_index: target_index as u64, + })?; + + Ok(()) } diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 80fdeee66e0..7cd22bba8f8 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -7,8 +7,7 @@ use ssz::Decode; use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache; use state_processing::epoch_cache::initialize_epoch_cache; use state_processing::per_block_processing::process_operations::{ - process_consolidation_requests, process_deposit_receipts, - process_execution_layer_withdrawal_requests, + process_deposit_requests, process_withdrawal_requests, }; use state_processing::{ per_block_processing::{ @@ -464,7 +463,7 @@ impl Operation for WithdrawalRequest { spec: &ChainSpec, _extra: &Operations, ) -> Result<(), BlockProcessingError> { - process_execution_layer_withdrawal_requests(state, &[self.clone()], spec) + process_withdrawal_requests(state, &[self.clone()], spec) } } @@ -487,7 +486,7 @@ impl Operation for DepositRequest { spec: &ChainSpec, _extra: &Operations, ) -> Result<(), BlockProcessingError> { - process_deposit_receipts(state, &[self.clone()], spec) + process_deposit_requests(state, &[self.clone()], spec) } } From 86af44070cf355b2225c61fad51c906223cf0bfd Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 26 Jun 2024 09:36:09 -0700 Subject: [PATCH 185/227] add light client ssz tests --- testing/ef_tests/src/type_name.rs | 15 +++++++++++++-- testing/ef_tests/tests/tests.rs | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 42c8d6035dd..dd12dacafa2 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -90,6 +90,7 @@ type_name_generic!(LightClientBootstrap); type_name_generic!(LightClientBootstrapAltair, "LightClientBootstrap"); type_name_generic!(LightClientBootstrapCapella, "LightClientBootstrap"); type_name_generic!(LightClientBootstrapDeneb, "LightClientBootstrap"); +type_name_generic!(LightClientBootstrapElectra, "LightClientBootstrap"); type_name_generic!(LightClientFinalityUpdate); type_name_generic!(LightClientFinalityUpdateAltair, "LightClientFinalityUpdate"); type_name_generic!( @@ -97,10 +98,15 @@ type_name_generic!( "LightClientFinalityUpdate" ); type_name_generic!(LightClientFinalityUpdateDeneb, "LightClientFinalityUpdate"); +type_name_generic!( + LightClientFinalityUpdateElectra, + "LightClientFinalityUpdate" +); type_name_generic!(LightClientHeader); -type_name_generic!(LightClientHeaderDeneb, "LightClientHeader"); -type_name_generic!(LightClientHeaderCapella, "LightClientHeader"); type_name_generic!(LightClientHeaderAltair, "LightClientHeader"); +type_name_generic!(LightClientHeaderCapella, "LightClientHeader"); +type_name_generic!(LightClientHeaderDeneb, "LightClientHeader"); +type_name_generic!(LightClientHeaderElectra, "LightClientHeader"); type_name_generic!(LightClientOptimisticUpdate); type_name_generic!( LightClientOptimisticUpdateAltair, @@ -114,10 +120,15 @@ type_name_generic!( LightClientOptimisticUpdateDeneb, "LightClientOptimisticUpdate" ); +type_name_generic!( + LightClientOptimisticUpdateElectra, + "LightClientOptimisticUpdate" +); type_name_generic!(LightClientUpdate); type_name_generic!(LightClientUpdateAltair, "LightClientUpdate"); type_name_generic!(LightClientUpdateCapella, "LightClientUpdate"); type_name_generic!(LightClientUpdateDeneb, "LightClientUpdate"); +type_name_generic!(LightClientUpdateElectra, "LightClientUpdate"); type_name_generic!(PendingAttestation); type_name!(PendingConsolidation); type_name!(PendingPartialWithdrawal); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 80b5cb458ab..71d0b31b01e 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -396,6 +396,10 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only() + .run(); } // LightClientHeader has no internal indicator of which fork it is for, so we test it separately. @@ -421,6 +425,12 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); } // LightClientOptimisticUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -434,6 +444,8 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::capella_only().run(); SszStaticHandler::, MinimalEthSpec>::deneb_only().run(); SszStaticHandler::, MainnetEthSpec>::deneb_only().run(); + SszStaticHandler::, MinimalEthSpec>::electra_only().run(); + SszStaticHandler::, MainnetEthSpec>::electra_only().run(); } // LightClientFinalityUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -463,6 +475,12 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::deneb_only( ) .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); } // LightClientUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -486,6 +504,12 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); + SszStaticHandler::, MinimalEthSpec>::electra_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::electra_only( + ) + .run(); } #[test] From e1fdda4f04ed5327a9d1ab6c5c196d2274906c7c Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Wed, 26 Jun 2024 12:38:09 -0500 Subject: [PATCH 186/227] Update `engine-api` for `devnet-1` --- .../beacon_chain/src/beacon_block_streamer.rs | 4 +- beacon_node/execution_layer/src/engine_api.rs | 246 ++++++++++++------ .../execution_layer/src/engine_api/http.rs | 93 ++++++- .../src/engine_api/json_structures.rs | 85 +++--- beacon_node/execution_layer/src/lib.rs | 6 +- .../src/test_utils/handle_rpc.rs | 60 +++++ .../execution_layer/src/test_utils/mod.rs | 3 + .../types/src/execution_payload_header.rs | 9 + 8 files changed, 393 insertions(+), 113 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_block_streamer.rs b/beacon_node/beacon_chain/src/beacon_block_streamer.rs index f0a68b6be55..5e3ac6daa7c 100644 --- a/beacon_node/beacon_chain/src/beacon_block_streamer.rs +++ b/beacon_node/beacon_chain/src/beacon_block_streamer.rs @@ -1,5 +1,5 @@ use crate::{metrics, BeaconChain, BeaconChainError, BeaconChainTypes, BlockProcessStatus}; -use execution_layer::{ExecutionLayer, ExecutionPayloadBodyV1}; +use execution_layer::{ExecutionLayer, ExecutionPayloadBody}; use slog::{crit, debug, error, Logger}; use std::collections::HashMap; use std::sync::Arc; @@ -57,7 +57,7 @@ struct BodiesByRange { struct BlockParts { blinded_block: Box>, header: Box>, - body: Option>>, + body: Option>>, } impl BlockParts { diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 2391d8c087b..d5c00e4dbe6 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -2,7 +2,8 @@ use crate::engines::ForkchoiceState; use crate::http::{ ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_FORKCHOICE_UPDATED_V3, ENGINE_GET_CLIENT_VERSION_V1, ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, - ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, + ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V2, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, + ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V2, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, ENGINE_GET_PAYLOAD_V3, ENGINE_GET_PAYLOAD_V4, ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V4, }; @@ -20,7 +21,7 @@ use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use strum::IntoStaticStr; use superstruct::superstruct; -use types::execution_payload::{DepositRequests, WithdrawalRequests}; +use types::execution_payload::{ConsolidationRequests, DepositRequests, WithdrawalRequests}; pub use types::{ Address, BeaconBlockRef, ConsolidationRequest, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, @@ -547,27 +548,106 @@ impl GetPayloadResponse { } } +#[superstruct( + variants(V1, V2), + variant_attributes(derive(Clone, Debug),), + partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant") +)] #[derive(Clone, Debug)] -pub struct ExecutionPayloadBodyV1 { +pub struct ExecutionPayloadBody { pub transactions: Transactions, pub withdrawals: Option>, + #[superstruct(only(V2))] pub deposit_requests: Option>, + #[superstruct(only(V2))] pub withdrawal_requests: Option>, + #[superstruct(only(V2))] + pub consolidation_requests: Option>, } -impl ExecutionPayloadBodyV1 { +impl ExecutionPayloadBody { + #[allow(clippy::type_complexity)] + pub fn deconstruct( + self, + ) -> ( + Transactions, + Option>, + Option>, + Option>, + Option>, + ) { + match self { + ExecutionPayloadBody::V1(body) => { + (body.transactions, body.withdrawals, None, None, None) + } + ExecutionPayloadBody::V2(body) => ( + body.transactions, + body.withdrawals, + body.deposit_requests, + body.withdrawal_requests, + body.consolidation_requests, + ), + } + } pub fn to_payload( self, header: ExecutionPayloadHeader, ) -> Result, String> { - match header { - ExecutionPayloadHeader::Bellatrix(header) => { - if self.withdrawals.is_some() { + let header_fork = header.fork_name_unchecked(); + match &self { + Self::V1(_) => { + if header_fork.electra_enabled() { return Err(format!( - "block {} is merge but payload body has withdrawals", - header.block_hash + "block {} is {} but response is ExecutionPayloadBodyV1. Does the EL support {}?", + header.block_hash(), + header_fork, + ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V2, )); } + } + Self::V2(_) => {} + } + + let ( + transactions, + withdrawals, + deposit_requests, + withdrawal_requests, + consolidation_requests, + ) = self.deconstruct(); + if !header_fork.capella_enabled() && withdrawals.is_some() { + return Err(format!( + "block {} is {} but payload body has withdrawals", + header.block_hash(), + header_fork + )); + } + if !header_fork.electra_enabled() { + if deposit_requests.is_some() { + return Err(format!( + "block {} is {} but payload body has deposit_requests", + header.block_hash(), + header_fork + )); + } + if withdrawal_requests.is_some() { + return Err(format!( + "block {} is {} but payload body has withdrawal_requests", + header.block_hash(), + header_fork + )); + } + if consolidation_requests.is_some() { + return Err(format!( + "block {} is {} but payload body has consolidation_requests", + header.block_hash(), + header_fork + )); + } + } + + match header { + ExecutionPayloadHeader::Bellatrix(header) => { Ok(ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix { parent_hash: header.parent_hash, fee_recipient: header.fee_recipient, @@ -582,75 +662,86 @@ impl ExecutionPayloadBodyV1 { extra_data: header.extra_data, base_fee_per_gas: header.base_fee_per_gas, block_hash: header.block_hash, - transactions: self.transactions, + transactions, })) } ExecutionPayloadHeader::Capella(header) => { - if let Some(withdrawals) = self.withdrawals { - Ok(ExecutionPayload::Capella(ExecutionPayloadCapella { - parent_hash: header.parent_hash, - fee_recipient: header.fee_recipient, - state_root: header.state_root, - receipts_root: header.receipts_root, - logs_bloom: header.logs_bloom, - prev_randao: header.prev_randao, - block_number: header.block_number, - gas_limit: header.gas_limit, - gas_used: header.gas_used, - timestamp: header.timestamp, - extra_data: header.extra_data, - base_fee_per_gas: header.base_fee_per_gas, - block_hash: header.block_hash, - transactions: self.transactions, - withdrawals, - })) - } else { - Err(format!( - "block {} is capella but payload body doesn't have withdrawals", - header.block_hash - )) - } + let withdrawals = withdrawals.ok_or_else(|| { + format!( + "block {} is {} but payload body has withdrawals set to null", + header.block_hash, header_fork + ) + })?; + Ok(ExecutionPayload::Capella(ExecutionPayloadCapella { + parent_hash: header.parent_hash, + fee_recipient: header.fee_recipient, + state_root: header.state_root, + receipts_root: header.receipts_root, + logs_bloom: header.logs_bloom, + prev_randao: header.prev_randao, + block_number: header.block_number, + gas_limit: header.gas_limit, + gas_used: header.gas_used, + timestamp: header.timestamp, + extra_data: header.extra_data, + base_fee_per_gas: header.base_fee_per_gas, + block_hash: header.block_hash, + transactions, + withdrawals, + })) } ExecutionPayloadHeader::Deneb(header) => { - if let Some(withdrawals) = self.withdrawals { - Ok(ExecutionPayload::Deneb(ExecutionPayloadDeneb { - parent_hash: header.parent_hash, - fee_recipient: header.fee_recipient, - state_root: header.state_root, - receipts_root: header.receipts_root, - logs_bloom: header.logs_bloom, - prev_randao: header.prev_randao, - block_number: header.block_number, - gas_limit: header.gas_limit, - gas_used: header.gas_used, - timestamp: header.timestamp, - extra_data: header.extra_data, - base_fee_per_gas: header.base_fee_per_gas, - block_hash: header.block_hash, - transactions: self.transactions, - withdrawals, - blob_gas_used: header.blob_gas_used, - excess_blob_gas: header.excess_blob_gas, - })) - } else { - Err(format!( - "block {} is post-capella but payload body doesn't have withdrawals", - header.block_hash - )) - } + let withdrawals = withdrawals.ok_or_else(|| { + format!( + "block {} is {} but payload body has withdrawals set to null", + header.block_hash, header_fork + ) + })?; + Ok(ExecutionPayload::Deneb(ExecutionPayloadDeneb { + parent_hash: header.parent_hash, + fee_recipient: header.fee_recipient, + state_root: header.state_root, + receipts_root: header.receipts_root, + logs_bloom: header.logs_bloom, + prev_randao: header.prev_randao, + block_number: header.block_number, + gas_limit: header.gas_limit, + gas_used: header.gas_used, + timestamp: header.timestamp, + extra_data: header.extra_data, + base_fee_per_gas: header.base_fee_per_gas, + block_hash: header.block_hash, + transactions, + withdrawals, + blob_gas_used: header.blob_gas_used, + excess_blob_gas: header.excess_blob_gas, + })) } ExecutionPayloadHeader::Electra(header) => { - let (Some(withdrawals), Some(deposit_requests), Some(withdrawal_requests)) = ( - self.withdrawals, - self.deposit_requests, - self.withdrawal_requests, - ) else { - return Err(format!( - "block {} is post-electra but payload body doesn't have withdrawals/deposit_requests/withdrawal_requests \ - Check that ELs are returning receipts and withdrawal_requests in getPayloadBody requests", - header.block_hash - )); - }; + let withdrawals = withdrawals.ok_or_else(|| { + format!( + "block {} is {} but payload body has withdrawals set to null", + header.block_hash, header_fork + ) + })?; + let deposit_requests = deposit_requests.ok_or_else(|| { + format!( + "block {} is {} but payload body has deposit_requests set to null", + header.block_hash, header_fork + ) + })?; + let withdrawal_requests = withdrawal_requests.ok_or_else(|| { + format!( + "block {} is {} but payload body has withdrawal_requests set to null", + header.block_hash, header_fork + ) + })?; + let consolidation_requests = consolidation_requests.ok_or_else(|| { + format!( + "block {} is {} but payload body has consolidation_requests set to null", + header.block_hash, header_fork + ) + })?; Ok(ExecutionPayload::Electra(ExecutionPayloadElectra { parent_hash: header.parent_hash, fee_recipient: header.fee_recipient, @@ -665,14 +756,13 @@ impl ExecutionPayloadBodyV1 { extra_data: header.extra_data, base_fee_per_gas: header.base_fee_per_gas, block_hash: header.block_hash, - transactions: self.transactions, + transactions, withdrawals, blob_gas_used: header.blob_gas_used, excess_blob_gas: header.excess_blob_gas, deposit_requests, withdrawal_requests, - // TODO(electra): gonna need to superstruct this thing to do it properly.. - consolidation_requests: Vec::new().into(), + consolidation_requests, })) } } @@ -690,6 +780,8 @@ pub struct EngineCapabilities { pub forkchoice_updated_v3: bool, pub get_payload_bodies_by_hash_v1: bool, pub get_payload_bodies_by_range_v1: bool, + pub get_payload_bodies_by_hash_v2: bool, + pub get_payload_bodies_by_range_v2: bool, pub get_payload_v1: bool, pub get_payload_v2: bool, pub get_payload_v3: bool, @@ -727,6 +819,12 @@ impl EngineCapabilities { if self.get_payload_bodies_by_range_v1 { response.push(ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1); } + if self.get_payload_bodies_by_hash_v2 { + response.push(ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V2); + } + if self.get_payload_bodies_by_range_v2 { + response.push(ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V2); + } if self.get_payload_v1 { response.push(ENGINE_GET_PAYLOAD_V1); } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 1c03cc81fc3..73eabf574b5 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -50,6 +50,8 @@ pub const ENGINE_FORKCHOICE_UPDATED_TIMEOUT: Duration = Duration::from_secs(8); pub const ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1: &str = "engine_getPayloadBodiesByHashV1"; pub const ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1: &str = "engine_getPayloadBodiesByRangeV1"; +pub const ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V2: &str = "engine_getPayloadBodiesByHashV2"; +pub const ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V2: &str = "engine_getPayloadBodiesByRangeV2"; pub const ENGINE_GET_PAYLOAD_BODIES_TIMEOUT: Duration = Duration::from_secs(10); pub const ENGINE_EXCHANGE_CAPABILITIES: &str = "engine_exchangeCapabilities"; @@ -78,6 +80,8 @@ pub static LIGHTHOUSE_CAPABILITIES: &[&str] = &[ ENGINE_FORKCHOICE_UPDATED_V3, ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, + ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V2, + ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V2, ENGINE_GET_CLIENT_VERSION_V1, ]; @@ -1036,7 +1040,7 @@ impl HttpJsonRpc { pub async fn get_payload_bodies_by_hash_v1( &self, block_hashes: Vec, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let params = json!([block_hashes]); let response: Vec>> = self @@ -1049,7 +1053,27 @@ impl HttpJsonRpc { Ok(response .into_iter() - .map(|opt_json| opt_json.map(From::from)) + .map(|opt_json| opt_json.map(|v1| JsonExecutionPayloadBody::V1(v1).into())) + .collect()) + } + + pub async fn get_payload_bodies_by_hash_v2( + &self, + block_hashes: Vec, + ) -> Result>>, Error> { + let params = json!([block_hashes]); + + let response: Vec>> = self + .rpc_request( + ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V2, + params, + ENGINE_GET_PAYLOAD_BODIES_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + + Ok(response + .into_iter() + .map(|opt_json| opt_json.map(|v2| JsonExecutionPayloadBody::V2(v2).into())) .collect()) } @@ -1057,7 +1081,7 @@ impl HttpJsonRpc { &self, start: u64, count: u64, - ) -> Result>>, Error> { + ) -> Result>>, Error> { #[derive(Serialize)] #[serde(transparent)] struct Quantity(#[serde(with = "serde_utils::u64_hex_be")] u64); @@ -1073,7 +1097,31 @@ impl HttpJsonRpc { Ok(response .into_iter() - .map(|opt_json| opt_json.map(From::from)) + .map(|opt_json| opt_json.map(|v1| JsonExecutionPayloadBody::V1(v1).into())) + .collect()) + } + + pub async fn get_payload_bodies_by_range_v2( + &self, + start: u64, + count: u64, + ) -> Result>>, Error> { + #[derive(Serialize)] + #[serde(transparent)] + struct Quantity(#[serde(with = "serde_utils::u64_hex_be")] u64); + + let params = json!([Quantity(start), Quantity(count)]); + let response: Vec>> = self + .rpc_request( + ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V2, + params, + ENGINE_GET_PAYLOAD_BODIES_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + + Ok(response + .into_iter() + .map(|opt_json| opt_json.map(|v2| JsonExecutionPayloadBody::V1(v2).into())) .collect()) } @@ -1100,6 +1148,10 @@ impl HttpJsonRpc { .contains(ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1), get_payload_bodies_by_range_v1: capabilities .contains(ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1), + get_payload_bodies_by_hash_v2: capabilities + .contains(ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V2), + get_payload_bodies_by_range_v2: capabilities + .contains(ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V2), get_payload_v1: capabilities.contains(ENGINE_GET_PAYLOAD_V1), get_payload_v2: capabilities.contains(ENGINE_GET_PAYLOAD_V2), get_payload_v3: capabilities.contains(ENGINE_GET_PAYLOAD_V3), @@ -1275,6 +1327,39 @@ impl HttpJsonRpc { } } + pub async fn get_payload_bodies_by_hash( + &self, + block_hashes: Vec, + ) -> Result>>, Error> { + let engine_capabilities = self.get_engine_capabilities(None).await?; + if engine_capabilities.get_payload_bodies_by_hash_v2 { + self.get_payload_bodies_by_hash_v2(block_hashes).await + } else if engine_capabilities.get_payload_bodies_by_hash_v1 { + self.get_payload_bodies_by_hash_v1(block_hashes).await + } else { + Err(Error::RequiredMethodUnsupported( + "engine_getPayloadBodiesByHash", + )) + } + } + + pub async fn get_payload_bodies_by_range( + &self, + start: u64, + count: u64, + ) -> Result>>, Error> { + let engine_capabilities = self.get_engine_capabilities(None).await?; + if engine_capabilities.get_payload_bodies_by_range_v2 { + self.get_payload_bodies_by_range_v2(start, count).await + } else if engine_capabilities.get_payload_bodies_by_range_v1 { + self.get_payload_bodies_by_range_v1(start, count).await + } else { + Err(Error::RequiredMethodUnsupported( + "engine_getPayloadBodiesByRange", + )) + } + } + // automatically selects the latest version of // forkchoice_updated that the execution engine supports pub async fn forkchoice_updated( diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index b5f04fed19e..0f566c8061e 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -719,45 +719,70 @@ impl From for JsonForkchoiceUpdatedV1Response { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[superstruct( + variants(V1, V2), + variant_attributes(derive(Clone, Debug, Serialize, Deserialize),), + partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant") +)] +#[derive(Clone, Debug, Serialize)] #[serde(bound = "E: EthSpec")] #[serde(rename_all = "camelCase")] -pub struct JsonExecutionPayloadBodyV1 { +#[serde(untagged)] +pub struct JsonExecutionPayloadBody { #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, pub withdrawals: Option>, + #[superstruct(only(V2))] pub deposit_requests: Option>, + #[superstruct(only(V2))] pub withdrawal_requests: Option>, -} - -impl From> for ExecutionPayloadBodyV1 { - fn from(value: JsonExecutionPayloadBodyV1) -> Self { - Self { - transactions: value.transactions, - withdrawals: value.withdrawals.map(|json_withdrawals| { - Withdrawals::::from( - json_withdrawals - .into_iter() - .map(Into::into) - .collect::>(), - ) - }), - deposit_requests: value.deposit_requests.map(|json_receipts| { - DepositRequests::::from( - json_receipts - .into_iter() - .map(Into::into) - .collect::>(), - ) + #[superstruct(only(V2))] + pub consolidation_requests: + Option>, +} + +impl From> for ExecutionPayloadBody { + fn from(value: JsonExecutionPayloadBody) -> Self { + match value { + JsonExecutionPayloadBody::V1(body_v1) => Self::V1(ExecutionPayloadBodyV1 { + transactions: body_v1.transactions, + withdrawals: body_v1.withdrawals.map(|json_withdrawals| { + Withdrawals::::from( + json_withdrawals + .into_iter() + .map(Into::into) + .collect::>(), + ) + }), }), - withdrawal_requests: value.withdrawal_requests.map(|json_withdrawal_requests| { - WithdrawalRequests::::from( - json_withdrawal_requests - .into_iter() - .map(Into::into) - .collect::>(), - ) + JsonExecutionPayloadBody::V2(body_v2) => Self::V2(ExecutionPayloadBodyV2 { + transactions: body_v2.transactions, + withdrawals: body_v2.withdrawals.map(|json_withdrawals| { + Withdrawals::::from( + json_withdrawals + .into_iter() + .map(Into::into) + .collect::>(), + ) + }), + deposit_requests: body_v2.deposit_requests.map(|json_receipts| { + DepositRequests::::from( + json_receipts + .into_iter() + .map(Into::into) + .collect::>(), + ) + }), + withdrawal_requests: body_v2.withdrawal_requests.map(|json_withdrawal_requests| { + WithdrawalRequests::::from( + json_withdrawal_requests + .into_iter() + .map(Into::into) + .collect::>(), + ) + }), + consolidation_requests: body_v2.consolidation_requests, }), } } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index cd41fc97a10..e40e948cc6b 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1771,10 +1771,10 @@ impl ExecutionLayer { pub async fn get_payload_bodies_by_hash( &self, hashes: Vec, - ) -> Result>>, Error> { + ) -> Result>>, Error> { self.engine() .request(|engine: &Engine| async move { - engine.api.get_payload_bodies_by_hash_v1(hashes).await + engine.api.get_payload_bodies_by_hash(hashes).await }) .await .map_err(Box::new) @@ -1785,7 +1785,7 @@ impl ExecutionLayer { &self, start: u64, count: u64, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let _timer = metrics::start_timer(&metrics::EXECUTION_LAYER_GET_PAYLOAD_BODIES_BY_RANGE); self.engine() .request(|engine: &Engine| async move { diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 0dc7a7759c5..ca379af7108 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -589,6 +589,61 @@ pub async fn handle_rpc( .withdrawals() .ok() .map(|withdrawals| VariableList::from(withdrawals.clone())), + })); + } + None => response.push(None), + } + } + + Ok(serde_json::to_value(response).unwrap()) + } + ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V2 => { + #[derive(Deserialize)] + #[serde(transparent)] + struct Quantity(#[serde(with = "serde_utils::u64_hex_be")] pub u64); + + let start = get_param::(params, 0) + .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))? + .0; + let count = get_param::(params, 1) + .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))? + .0; + + let mut response = vec![]; + for block_num in start..(start + count) { + let maybe_block = ctx + .execution_block_generator + .read() + .execution_block_with_txs_by_number(block_num); + + match maybe_block { + Some(block) => { + let transactions = Transactions::::new( + block + .transactions() + .iter() + .map(|transaction| VariableList::new(transaction.rlp().to_vec())) + .collect::>() + .map_err(|e| { + ( + format!("failed to deserialize transaction: {:?}", e), + GENERIC_ERROR_CODE, + ) + })?, + ) + .map_err(|e| { + ( + format!("failed to deserialize transactions: {:?}", e), + GENERIC_ERROR_CODE, + ) + })?; + + response.push(Some(JsonExecutionPayloadBodyV2:: { + transactions, + withdrawals: block + .withdrawals() + .ok() + .map(|withdrawals| VariableList::from(withdrawals.clone())), deposit_requests: block.deposit_requests().ok().map( |deposit_requests| VariableList::from(deposit_requests.clone()), ), @@ -597,6 +652,11 @@ pub async fn handle_rpc( VariableList::from(withdrawal_requests.clone()) }, ), + consolidation_requests: block.consolidation_requests().ok().map( + |consolidation_requests| { + VariableList::from(consolidation_requests.clone()) + }, + ), })); } None => response.push(None), diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 7b00ca9fbc3..25f9a982be9 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -48,7 +48,10 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { forkchoice_updated_v2: true, forkchoice_updated_v3: true, get_payload_bodies_by_hash_v1: true, + // TODO(electra): make a v2 implementation for the mock EL + get_payload_bodies_by_hash_v2: false, get_payload_bodies_by_range_v1: true, + get_payload_bodies_by_range_v2: false, get_payload_v1: true, get_payload_v2: true, get_payload_v3: true, diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 477d4f5d640..28bbfb9c048 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -127,6 +127,15 @@ impl ExecutionPayloadHeader { } } } + + pub fn fork_name_unchecked(&self) -> ForkName { + match self { + ExecutionPayloadHeader::Bellatrix(_) => ForkName::Bellatrix, + ExecutionPayloadHeader::Capella(_) => ForkName::Capella, + ExecutionPayloadHeader::Deneb(_) => ForkName::Deneb, + ExecutionPayloadHeader::Electra(_) => ForkName::Electra, + } + } } impl<'a, E: EthSpec> ExecutionPayloadHeaderRef<'a, E> { From 6e9895441123a2441a53810848420c131d54aba3 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 26 Jun 2024 10:54:28 -0700 Subject: [PATCH 187/227] disable failing light client ssz tests --- testing/ef_tests/check_all_files_accessed.py | 6 +++- testing/ef_tests/tests/tests.rs | 35 +++++++++++--------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index e4f0745169d..9be9d9dbed5 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -43,7 +43,11 @@ "bls12-381-tests/hash_to_G2", "tests/.*/eip6110", "tests/.*/whisk", - "tests/.*/eip7594" + "tests/.*/eip7594", + # re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved + "tests/.*/.*/electra/ssz_static/LightClientUpdate", + "tests/.*/.*/electra/ssz_static/LightClientFinalityUpdate", + "tests/.*/.*/electra/ssz_static/LightClientBootstrap" ] diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 71d0b31b01e..7a7c264682e 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -396,10 +396,11 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only() - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only() - .run(); + // Re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved + // SszStaticHandler::, MinimalEthSpec>::electra_only() + // .run(); + // SszStaticHandler::, MainnetEthSpec>::electra_only() + // .run(); } // LightClientHeader has no internal indicator of which fork it is for, so we test it separately. @@ -475,12 +476,13 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::deneb_only( ) .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( - ) - .run(); + // Re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved + // SszStaticHandler::, MinimalEthSpec>::electra_only( + // ) + // .run(); + // SszStaticHandler::, MainnetEthSpec>::electra_only( + // ) + // .run(); } // LightClientUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -504,12 +506,13 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( - ) - .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( - ) - .run(); + // Re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved + // SszStaticHandler::, MinimalEthSpec>::electra_only( + // ) + // .run(); + // SszStaticHandler::, MainnetEthSpec>::electra_only( + // ) + // .run(); } #[test] From 50d0b2f5e6b55be1369cfa5b4369e7d6158f5d48 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 26 Jun 2024 11:01:10 -0700 Subject: [PATCH 188/227] correctly exclude light client struct from accessed files --- testing/ef_tests/check_all_files_accessed.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index 9be9d9dbed5..d0a19699d58 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -45,9 +45,9 @@ "tests/.*/whisk", "tests/.*/eip7594", # re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved - "tests/.*/.*/electra/ssz_static/LightClientUpdate", - "tests/.*/.*/electra/ssz_static/LightClientFinalityUpdate", - "tests/.*/.*/electra/ssz_static/LightClientBootstrap" + "tests/.*/electra/ssz_static/LightClientUpdate", + "tests/.*/electra/ssz_static/LightClientFinalityUpdate", + "tests/.*/electra/ssz_static/LightClientBootstrap" ] From 9436b40dc1e535f048133ad47136e980948d7f55 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Wed, 26 Jun 2024 14:26:16 -0500 Subject: [PATCH 189/227] Fix Failing Tests --- beacon_node/execution_layer/src/test_utils/handle_rpc.rs | 4 ++++ beacon_node/execution_layer/src/test_utils/mod.rs | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index ca379af7108..d86b4439ad7 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -638,6 +638,10 @@ pub async fn handle_rpc( ) })?; + // TODO(electra): add testing for: + // deposit_requests + // withdrawal_requests + // consolidation_requests response.push(Some(JsonExecutionPayloadBodyV2:: { transactions, withdrawals: block diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 25f9a982be9..f883f162cdc 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -48,10 +48,9 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { forkchoice_updated_v2: true, forkchoice_updated_v3: true, get_payload_bodies_by_hash_v1: true, - // TODO(electra): make a v2 implementation for the mock EL - get_payload_bodies_by_hash_v2: false, + get_payload_bodies_by_hash_v2: true, get_payload_bodies_by_range_v1: true, - get_payload_bodies_by_range_v2: false, + get_payload_bodies_by_range_v2: true, get_payload_v1: true, get_payload_v2: true, get_payload_v3: true, From 95939dcf0f7c7d81749aeac9d4651507744087ea Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:53:54 -0500 Subject: [PATCH 190/227] Fix Calculations Relying on MAX_EB (#5983) * Fix Calculations Relying on MAX_EB * Use spec fn for max_effective_balance --- consensus/types/src/beacon_state.rs | 12 ++++++------ consensus/types/src/validator.rs | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 295aaade1cf..8642036ad61 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -892,6 +892,8 @@ impl BeaconState { return Err(Error::InsufficientValidators); } + let max_effective_balance = spec.max_effective_balance_for_fork(self.fork_name_unchecked()); + let mut i = 0; loop { let shuffled_index = compute_shuffled_index( @@ -907,9 +909,7 @@ impl BeaconState { let random_byte = Self::shuffling_random_byte(i, seed)?; let effective_balance = self.get_effective_balance(candidate_index)?; if effective_balance.safe_mul(MAX_RANDOM_BYTE)? - >= spec - .max_effective_balance - .safe_mul(u64::from(random_byte))? + >= max_effective_balance.safe_mul(u64::from(random_byte))? { return Ok(candidate_index); } @@ -1091,6 +1091,8 @@ impl BeaconState { let seed = self.get_seed(epoch, Domain::SyncCommittee, spec)?; + let max_effective_balance = spec.max_effective_balance_for_fork(self.fork_name_unchecked()); + let mut i = 0; let mut sync_committee_indices = Vec::with_capacity(E::SyncCommitteeSize::to_usize()); while sync_committee_indices.len() < E::SyncCommitteeSize::to_usize() { @@ -1107,9 +1109,7 @@ impl BeaconState { let random_byte = Self::shuffling_random_byte(i, seed.as_bytes())?; let effective_balance = self.get_validator(candidate_index)?.effective_balance; if effective_balance.safe_mul(MAX_RANDOM_BYTE)? - >= spec - .max_effective_balance - .safe_mul(u64::from(random_byte))? + >= max_effective_balance.safe_mul(u64::from(random_byte))? { sync_committee_indices.push(candidate_index); } diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index b5e92d1f5d8..4173e18526e 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -219,6 +219,7 @@ impl Validator { } } + /// TODO(electra): refactor these functions and make it simpler.. this is a mess /// Returns `true` if the validator is partially withdrawable. fn is_partially_withdrawable_validator_capella(&self, balance: u64, spec: &ChainSpec) -> bool { self.has_eth1_withdrawal_credential(spec) From 806a5ebe1fdec69a730a1251acb5daf756819068 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Fri, 28 Jun 2024 14:53:10 +0530 Subject: [PATCH 191/227] The great renaming receipt -> request --- beacon_node/execution_layer/src/engine_api.rs | 20 ++++++------ .../src/engine_api/json_structures.rs | 20 ++++++------ beacon_node/execution_layer/src/lib.rs | 8 ++--- .../test_utils/execution_block_generator.rs | 2 +- .../src/test_utils/handle_rpc.rs | 4 +-- beacon_node/store/src/partial_beacon_state.rs | 6 ++-- .../process_operations.rs | 16 +++++----- .../state_processing/src/upgrade/electra.rs | 2 +- consensus/types/presets/gnosis/electra.yaml | 2 +- consensus/types/presets/mainnet/electra.yaml | 2 +- consensus/types/presets/minimal/electra.yaml | 2 +- consensus/types/src/beacon_state.rs | 2 +- consensus/types/src/chain_spec.rs | 7 ++-- consensus/types/src/config_and_preset.rs | 2 +- consensus/types/src/deposit_receipt.rs | 4 +-- consensus/types/src/eth_spec.rs | 14 ++++---- consensus/types/src/execution_payload.rs | 6 ++-- .../types/src/execution_payload_header.rs | 2 +- consensus/types/src/lib.rs | 2 +- consensus/types/src/payload.rs | 32 +++++++++---------- consensus/types/src/preset.rs | 4 +-- 21 files changed, 80 insertions(+), 79 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 6bfd9997f43..e0bfb7d49eb 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -20,7 +20,7 @@ use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use strum::IntoStaticStr; use superstruct::superstruct; -use types::execution_payload::{DepositReceipts, WithdrawalRequests}; +use types::execution_payload::{DepositRequests, WithdrawalRequests}; pub use types::{ Address, BeaconBlockRef, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, @@ -67,7 +67,7 @@ pub enum Error { TransitionConfigurationMismatch, SszError(ssz_types::Error), DeserializeWithdrawals(ssz_types::Error), - DeserializeDepositReceipts(ssz_types::Error), + DeserializeDepositRequests(ssz_types::Error), DeserializeWithdrawalRequests(ssz_types::Error), BuilderApi(builder_client::Error), IncorrectStateVariant, @@ -204,7 +204,7 @@ pub struct ExecutionBlockWithTransactions { #[serde(with = "serde_utils::u64_hex_be")] pub excess_blob_gas: u64, #[superstruct(only(Electra))] - pub deposit_receipts: Vec, + pub deposit_requests: Vec, #[superstruct(only(Electra))] pub withdrawal_requests: Vec, } @@ -314,8 +314,8 @@ impl TryFrom> for ExecutionBlockWithTransactions .collect(), blob_gas_used: block.blob_gas_used, excess_blob_gas: block.excess_blob_gas, - deposit_receipts: block - .deposit_receipts + deposit_requests: block + .deposit_requests .into_iter() .map(|deposit| deposit.into()) .collect(), @@ -546,7 +546,7 @@ impl GetPayloadResponse { pub struct ExecutionPayloadBodyV1 { pub transactions: Transactions, pub withdrawals: Option>, - pub deposit_receipts: Option>, + pub deposit_requests: Option>, pub withdrawal_requests: Option>, } @@ -635,13 +635,13 @@ impl ExecutionPayloadBodyV1 { } } ExecutionPayloadHeader::Electra(header) => { - let (Some(withdrawals), Some(deposit_receipts), Some(withdrawal_requests)) = ( + let (Some(withdrawals), Some(deposit_requests), Some(withdrawal_requests)) = ( self.withdrawals, - self.deposit_receipts, + self.deposit_requests, self.withdrawal_requests, ) else { return Err(format!( - "block {} is post-electra but payload body doesn't have withdrawals/deposit_receipts/withdrawal_requests \ + "block {} is post-electra but payload body doesn't have withdrawals/deposit_requests/withdrawal_requests \ Check that ELs are returning receipts and withdrawal_requests in getPayloadBody requests", header.block_hash )); @@ -664,7 +664,7 @@ impl ExecutionPayloadBodyV1 { withdrawals, blob_gas_used: header.blob_gas_used, excess_blob_gas: header.excess_blob_gas, - deposit_receipts, + deposit_requests, withdrawal_requests, })) } diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index c41ee30e3d9..fbffc47e29e 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -5,7 +5,7 @@ use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::BlobsList; use types::{ - DepositReceipt, ExecutionLayerWithdrawalRequest, FixedVector, PublicKeyBytes, Signature, + DepositRequest, ExecutionLayerWithdrawalRequest, FixedVector, PublicKeyBytes, Signature, Unsigned, }; @@ -106,7 +106,7 @@ pub struct JsonExecutionPayload { pub excess_blob_gas: u64, #[superstruct(only(V4))] // TODO(electra): Field name should be changed post devnet-0. See https://github.com/ethereum/execution-apis/pull/544 - pub deposit_requests: VariableList, + pub deposit_requests: VariableList, #[superstruct(only(V4))] pub withdrawal_requests: VariableList, @@ -213,7 +213,7 @@ impl From> for JsonExecutionPayloadV4 blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, deposit_requests: payload - .deposit_receipts + .deposit_requests .into_iter() .map(Into::into) .collect::>() @@ -340,7 +340,7 @@ impl From> for ExecutionPayloadElectra .into(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, - deposit_receipts: payload + deposit_requests: payload .deposit_requests .into_iter() .map(Into::into) @@ -725,7 +725,7 @@ pub struct JsonExecutionPayloadBodyV1 { #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, pub withdrawals: Option>, - pub deposit_receipts: Option>, + pub deposit_requests: Option>, pub withdrawal_requests: Option>, } @@ -742,8 +742,8 @@ impl From> for ExecutionPayloadBodyV1< .collect::>(), ) }), - deposit_receipts: value.deposit_receipts.map(|json_receipts| { - DepositReceipts::::from( + deposit_requests: value.deposit_requests.map(|json_receipts| { + DepositRequests::::from( json_receipts .into_iter() .map(Into::into) @@ -849,8 +849,8 @@ pub struct JsonDepositRequest { pub index: u64, } -impl From for JsonDepositRequest { - fn from(deposit: DepositReceipt) -> Self { +impl From for JsonDepositRequest { + fn from(deposit: DepositRequest) -> Self { Self { pubkey: deposit.pubkey, withdrawal_credentials: deposit.withdrawal_credentials, @@ -861,7 +861,7 @@ impl From for JsonDepositRequest { } } -impl From for DepositReceipt { +impl From for DepositRequest { fn from(json_deposit: JsonDepositRequest) -> Self { Self { pubkey: json_deposit.pubkey, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 6ac4b6eb9a5..eaa739d7a5d 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1994,14 +1994,14 @@ impl ExecutionLayer { .collect(), ) .map_err(ApiError::DeserializeWithdrawals)?; - let deposit_receipts = VariableList::new( + let deposit_requests = VariableList::new( electra_block - .deposit_receipts + .deposit_requests .into_iter() .map(Into::into) .collect(), ) - .map_err(ApiError::DeserializeDepositReceipts)?; + .map_err(ApiError::DeserializeDepositRequests)?; let withdrawal_requests = VariableList::new( electra_block .withdrawal_requests @@ -2028,7 +2028,7 @@ impl ExecutionLayer { withdrawals, blob_gas_used: electra_block.blob_gas_used, excess_blob_gas: electra_block.excess_blob_gas, - deposit_receipts, + deposit_requests, withdrawal_requests, }) } diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index e80c6b23705..8619e24a238 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -659,7 +659,7 @@ impl ExecutionBlockGenerator { withdrawals: pa.withdrawals.clone().into(), blob_gas_used: 0, excess_blob_gas: 0, - deposit_receipts: vec![].into(), + deposit_requests: vec![].into(), withdrawal_requests: vec![].into(), }), _ => unreachable!(), diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 9f1f244ef9a..0dc7a7759c5 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -589,8 +589,8 @@ pub async fn handle_rpc( .withdrawals() .ok() .map(|withdrawals| VariableList::from(withdrawals.clone())), - deposit_receipts: block.deposit_receipts().ok().map( - |deposit_receipts| VariableList::from(deposit_receipts.clone()), + deposit_requests: block.deposit_requests().ok().map( + |deposit_requests| VariableList::from(deposit_requests.clone()), ), withdrawal_requests: block.withdrawal_requests().ok().map( |withdrawal_requests| { diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index 5e6054bc06b..8f40b4b9241 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -121,7 +121,7 @@ where // Electra #[superstruct(only(Electra))] - pub deposit_receipts_start_index: u64, + pub deposit_requests_start_index: u64, #[superstruct(only(Electra))] pub deposit_balance_to_consume: u64, #[superstruct(only(Electra))] @@ -284,7 +284,7 @@ impl PartialBeaconState { latest_execution_payload_header, next_withdrawal_index, next_withdrawal_validator_index, - deposit_receipts_start_index, + deposit_requests_start_index, deposit_balance_to_consume, exit_balance_to_consume, earliest_exit_epoch, @@ -557,7 +557,7 @@ impl TryInto> for PartialBeaconState { latest_execution_payload_header, next_withdrawal_index, next_withdrawal_validator_index, - deposit_receipts_start_index, + deposit_requests_start_index, deposit_balance_to_consume, exit_balance_to_consume, earliest_exit_epoch, diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 02cf1d942cc..a2e576cf74e 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -44,9 +44,9 @@ pub fn process_operations>( if let Some(requests) = requests { process_execution_layer_withdrawal_requests(state, &requests, spec)?; } - let receipts = block_body.execution_payload()?.deposit_receipts()?; + let receipts = block_body.execution_payload()?.deposit_requests()?; if let Some(receipts) = receipts { - process_deposit_receipts(state, &receipts, spec)?; + process_deposit_requests(state, &receipts, spec)?; } process_consolidations(state, block_body.consolidations()?, verify_signatures, spec)?; } @@ -372,9 +372,9 @@ pub fn process_deposits( // [Modified in Electra:EIP6110] // Disable former deposit mechanism once all prior deposits are processed // - // If `deposit_receipts_start_index` does not exist as a field on `state`, electra is disabled + // If `deposit_requests_start_index` does not exist as a field on `state`, electra is disabled // which means we always want to use the old check, so this field defaults to `u64::MAX`. - let eth1_deposit_index_limit = state.deposit_receipts_start_index().unwrap_or(u64::MAX); + let eth1_deposit_index_limit = state.deposit_requests_start_index().unwrap_or(u64::MAX); if state.eth1_deposit_index() < eth1_deposit_index_limit { let expected_deposit_len = std::cmp::min( @@ -625,15 +625,15 @@ pub fn process_execution_layer_withdrawal_requests( Ok(()) } -pub fn process_deposit_receipts( +pub fn process_deposit_requests( state: &mut BeaconState, - receipts: &[DepositReceipt], + receipts: &[DepositRequest], spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { for receipt in receipts { // Set deposit receipt start index - if state.deposit_receipts_start_index()? == spec.unset_deposit_receipts_start_index { - *state.deposit_receipts_start_index_mut()? = receipt.index + if state.deposit_requests_start_index()? == spec.unset_deposit_requests_start_index { + *state.deposit_requests_start_index_mut()? = receipt.index } let deposit_data = DepositData { pubkey: receipt.pubkey, diff --git a/consensus/state_processing/src/upgrade/electra.rs b/consensus/state_processing/src/upgrade/electra.rs index 1e60bf488db..03cd96aebc5 100644 --- a/consensus/state_processing/src/upgrade/electra.rs +++ b/consensus/state_processing/src/upgrade/electra.rs @@ -78,7 +78,7 @@ pub fn upgrade_to_electra( next_withdrawal_validator_index: pre.next_withdrawal_validator_index, historical_summaries: pre.historical_summaries.clone(), // Electra - deposit_receipts_start_index: spec.unset_deposit_receipts_start_index, + deposit_requests_start_index: spec.unset_deposit_requests_start_index, deposit_balance_to_consume: 0, exit_balance_to_consume: 0, earliest_exit_epoch, diff --git a/consensus/types/presets/gnosis/electra.yaml b/consensus/types/presets/gnosis/electra.yaml index 72c626ded2f..38f6960bac8 100644 --- a/consensus/types/presets/gnosis/electra.yaml +++ b/consensus/types/presets/gnosis/electra.yaml @@ -35,7 +35,7 @@ MAX_CONSOLIDATIONS: 1 # Execution # --------------------------------------------------------------- # 2**13 (= 8192) receipts -MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192 +MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: 8192 # 2**4 (= 16) withdrawal requests MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16 diff --git a/consensus/types/presets/mainnet/electra.yaml b/consensus/types/presets/mainnet/electra.yaml index 72c626ded2f..38f6960bac8 100644 --- a/consensus/types/presets/mainnet/electra.yaml +++ b/consensus/types/presets/mainnet/electra.yaml @@ -35,7 +35,7 @@ MAX_CONSOLIDATIONS: 1 # Execution # --------------------------------------------------------------- # 2**13 (= 8192) receipts -MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192 +MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: 8192 # 2**4 (= 16) withdrawal requests MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16 diff --git a/consensus/types/presets/minimal/electra.yaml b/consensus/types/presets/minimal/electra.yaml index 11aa5e1f50e..cf726e004b1 100644 --- a/consensus/types/presets/minimal/electra.yaml +++ b/consensus/types/presets/minimal/electra.yaml @@ -35,7 +35,7 @@ MAX_CONSOLIDATIONS: 1 # Execution # --------------------------------------------------------------- # [customized] -MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4 +MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: 4 # [customized] 2**1 (= 2) withdrawal requests MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 2 diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 0426d43cacb..45f9178ff3f 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -481,7 +481,7 @@ where #[superstruct(only(Electra), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] #[serde(with = "serde_utils::quoted_u64")] - pub deposit_receipts_start_index: u64, + pub deposit_requests_start_index: u64, #[superstruct(only(Electra), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 0972e86a3f6..a9d521d9958 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -180,7 +180,8 @@ pub struct ChainSpec { pub electra_fork_version: [u8; 4], /// The Electra fork epoch is optional, with `None` representing "Electra never happens". pub electra_fork_epoch: Option, - pub unset_deposit_receipts_start_index: u64, + // TODO(pawan): this shouldn't be a config parameter? + pub unset_deposit_requests_start_index: u64, pub full_exit_request_amount: u64, pub min_activation_balance: u64, pub max_effective_balance_electra: u64, @@ -747,7 +748,7 @@ impl ChainSpec { */ electra_fork_version: [0x05, 00, 00, 00], electra_fork_epoch: None, - unset_deposit_receipts_start_index: u64::MAX, + unset_deposit_requests_start_index: u64::MAX, full_exit_request_amount: 0, min_activation_balance: option_wrapper(|| { u64::checked_pow(2, 5)?.checked_mul(u64::checked_pow(10, 9)?) @@ -1049,7 +1050,7 @@ impl ChainSpec { */ electra_fork_version: [0x05, 0x00, 0x00, 0x64], electra_fork_epoch: None, - unset_deposit_receipts_start_index: u64::MAX, + unset_deposit_requests_start_index: u64::MAX, full_exit_request_amount: 0, min_activation_balance: option_wrapper(|| { u64::checked_pow(2, 5)?.checked_mul(u64::checked_pow(10, 9)?) diff --git a/consensus/types/src/config_and_preset.rs b/consensus/types/src/config_and_preset.rs index 6fc6e0642ea..110392d4b77 100644 --- a/consensus/types/src/config_and_preset.rs +++ b/consensus/types/src/config_and_preset.rs @@ -124,7 +124,7 @@ pub fn get_extra_fields(spec: &ChainSpec) -> HashMap { "versioned_hash_version_kzg".to_uppercase() => deneb::VERSIONED_HASH_VERSION_KZG.to_string().into(), // Electra "compounding_withdrawal_prefix".to_uppercase() => u8_hex(spec.compounding_withdrawal_prefix_byte), - "unset_deposit_receipts_start_index".to_uppercase() => spec.unset_deposit_receipts_start_index.to_string().into(), + "unset_deposit_requests_start_index".to_uppercase() => spec.unset_deposit_requests_start_index.to_string().into(), "full_exit_request_amount".to_uppercase() => spec.full_exit_request_amount.to_string().into(), "domain_consolidation".to_uppercase()=> u32_hex(spec.domain_consolidation), } diff --git a/consensus/types/src/deposit_receipt.rs b/consensus/types/src/deposit_receipt.rs index 6a08f717f3d..f6ddf8b63a8 100644 --- a/consensus/types/src/deposit_receipt.rs +++ b/consensus/types/src/deposit_receipt.rs @@ -19,7 +19,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] -pub struct DepositReceipt { +pub struct DepositRequest { pub pubkey: PublicKeyBytes, pub withdrawal_credentials: Hash256, #[serde(with = "serde_utils::quoted_u64")] @@ -33,5 +33,5 @@ pub struct DepositReceipt { mod tests { use super::*; - ssz_and_tree_hash_tests!(DepositReceipt); + ssz_and_tree_hash_tests!(DepositRequest); } diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 1c379f5de42..c6e816a4875 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -144,7 +144,7 @@ pub trait EthSpec: type PendingPartialWithdrawalsLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; type PendingConsolidationsLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxConsolidations: Unsigned + Clone + Sync + Send + Debug + PartialEq; - type MaxDepositReceiptsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxDepositRequestsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxAttesterSlashingsElectra: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxAttestationsElectra: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxWithdrawalRequestsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; @@ -330,9 +330,9 @@ pub trait EthSpec: Self::MaxConsolidations::to_usize() } - /// Returns the `MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD` constant for this specification. - fn max_deposit_receipts_per_payload() -> usize { - Self::MaxDepositReceiptsPerPayload::to_usize() + /// Returns the `MAX_DEPOSIT_REQUESTS_PER_PAYLOAD` constant for this specification. + fn max_deposit_requests_per_payload() -> usize { + Self::MaxDepositRequestsPerPayload::to_usize() } /// Returns the `MAX_ATTESTER_SLASHINGS_ELECTRA` constant for this specification. @@ -405,7 +405,7 @@ impl EthSpec for MainnetEthSpec { type PendingPartialWithdrawalsLimit = U134217728; type PendingConsolidationsLimit = U262144; type MaxConsolidations = U1; - type MaxDepositReceiptsPerPayload = U8192; + type MaxDepositRequestsPerPayload = U8192; type MaxAttesterSlashingsElectra = U1; type MaxAttestationsElectra = U8; type MaxWithdrawalRequestsPerPayload = U16; @@ -442,7 +442,7 @@ impl EthSpec for MinimalEthSpec { type KzgCommitmentInclusionProofDepth = U9; type PendingPartialWithdrawalsLimit = U64; type PendingConsolidationsLimit = U64; - type MaxDepositReceiptsPerPayload = U4; + type MaxDepositRequestsPerPayload = U4; type MaxWithdrawalRequestsPerPayload = U2; params_from_eth_spec!(MainnetEthSpec { @@ -528,7 +528,7 @@ impl EthSpec for GnosisEthSpec { type PendingPartialWithdrawalsLimit = U134217728; type PendingConsolidationsLimit = U262144; type MaxConsolidations = U1; - type MaxDepositReceiptsPerPayload = U8192; + type MaxDepositRequestsPerPayload = U8192; type MaxAttesterSlashingsElectra = U1; type MaxAttestationsElectra = U8; type MaxWithdrawalRequestsPerPayload = U16; diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index cd6667b3751..eca561f7353 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -13,8 +13,8 @@ pub type Transactions = VariableList< >; pub type Withdrawals = VariableList::MaxWithdrawalsPerPayload>; -pub type DepositReceipts = - VariableList::MaxDepositReceiptsPerPayload>; +pub type DepositRequests = + VariableList::MaxDepositRequestsPerPayload>; pub type WithdrawalRequests = VariableList::MaxWithdrawalRequestsPerPayload>; @@ -94,7 +94,7 @@ pub struct ExecutionPayload { #[serde(with = "serde_utils::quoted_u64")] pub excess_blob_gas: u64, #[superstruct(only(Electra))] - pub deposit_receipts: VariableList, + pub deposit_requests: VariableList, #[superstruct(only(Electra))] pub withdrawal_requests: VariableList, diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index b8d0c8c2b0f..962e7a16fbc 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -295,7 +295,7 @@ impl<'a, E: EthSpec> From<&'a ExecutionPayloadElectra> for ExecutionPayloadHe withdrawals_root: payload.withdrawals.tree_hash_root(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, - deposit_receipts_root: payload.deposit_receipts.tree_hash_root(), + deposit_receipts_root: payload.deposit_requests.tree_hash_root(), withdrawal_requests_root: payload.withdrawal_requests.tree_hash_root(), } } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 6a98d7ade96..60c5a021e06 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -150,7 +150,7 @@ pub use crate::contribution_and_proof::ContributionAndProof; pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH}; pub use crate::deposit_data::DepositData; pub use crate::deposit_message::DepositMessage; -pub use crate::deposit_receipt::DepositReceipt; +pub use crate::deposit_receipt::DepositRequest; pub use crate::deposit_tree_snapshot::{DepositTreeSnapshot, FinalizedExecutionBlock}; pub use crate::enr_fork_id::EnrForkId; pub use crate::epoch_cache::{EpochCache, EpochCacheError, EpochCacheKey}; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 644d401ec7e..362cb6d3864 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -45,9 +45,9 @@ pub trait ExecPayload: Debug + Clone + PartialEq + Hash + TreeHash + Option>, Error, >; - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error>; + ) -> Result>, Error>; /// Is this a default payload with 0x0 roots for transactions and withdrawals? fn is_default_with_zero_roots(&self) -> bool; @@ -303,15 +303,15 @@ impl ExecPayload for FullPayload { } } - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error> { + ) -> Result>, Error> { match self { FullPayload::Bellatrix(_) | FullPayload::Capella(_) | FullPayload::Deneb(_) => { Err(Error::IncorrectStateVariant) } FullPayload::Electra(inner) => { - Ok(Some(inner.execution_payload.deposit_receipts.clone())) + Ok(Some(inner.execution_payload.deposit_requests.clone())) } } } @@ -464,15 +464,15 @@ impl<'b, E: EthSpec> ExecPayload for FullPayloadRef<'b, E> { } } - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error> { + ) -> Result>, Error> { match self { FullPayloadRef::Bellatrix(_) | FullPayloadRef::Capella(_) | FullPayloadRef::Deneb(_) => Err(Error::IncorrectStateVariant), FullPayloadRef::Electra(inner) => { - Ok(Some(inner.execution_payload.deposit_receipts.clone())) + Ok(Some(inner.execution_payload.deposit_requests.clone())) } } } @@ -666,9 +666,9 @@ impl ExecPayload for BlindedPayload { Ok(None) } - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error> { + ) -> Result>, Error> { Ok(None) } @@ -782,9 +782,9 @@ impl<'b, E: EthSpec> ExecPayload for BlindedPayloadRef<'b, E> { Ok(None) } - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error> { + ) -> Result>, Error> { Ok(None) } @@ -890,9 +890,9 @@ macro_rules! impl_exec_payload_common { i(self) } - fn deposit_receipts( + fn deposit_requests( &self, - ) -> Result>, Error> { + ) -> Result>, Error> { let j = $j; j(self) } @@ -1052,11 +1052,11 @@ macro_rules! impl_exec_payload_for_fork { let c: for<'a> fn( &'a $wrapper_type_full, ) -> Result< - Option>, + Option>, Error, > = |payload: &$wrapper_type_full| { let wrapper_ref_type = FullPayloadRef::$fork_variant(&payload); - wrapper_ref_type.deposit_receipts() + wrapper_ref_type.deposit_requests() }; c } diff --git a/consensus/types/src/preset.rs b/consensus/types/src/preset.rs index f4008d62e1d..9e9c5aaf410 100644 --- a/consensus/types/src/preset.rs +++ b/consensus/types/src/preset.rs @@ -248,7 +248,7 @@ pub struct ElectraPreset { #[serde(with = "serde_utils::quoted_u64")] pub max_consolidations: u64, #[serde(with = "serde_utils::quoted_u64")] - pub max_deposit_receipts_per_payload: u64, + pub max_deposit_requests_per_payload: u64, #[serde(with = "serde_utils::quoted_u64")] pub max_attester_slashings_electra: u64, #[serde(with = "serde_utils::quoted_u64")] @@ -270,7 +270,7 @@ impl ElectraPreset { pending_partial_withdrawals_limit: E::pending_partial_withdrawals_limit() as u64, pending_consolidations_limit: E::pending_consolidations_limit() as u64, max_consolidations: E::max_consolidations() as u64, - max_deposit_receipts_per_payload: E::max_deposit_receipts_per_payload() as u64, + max_deposit_requests_per_payload: E::max_deposit_requests_per_payload() as u64, max_attester_slashings_electra: E::max_attester_slashings_electra() as u64, max_attestations_electra: E::max_attestations_electra() as u64, max_withdrawal_requests_per_payload: E::max_withdrawal_requests_per_payload() as u64, From 033457ce899d13d1f4038ef3186548c54779d9df Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Fri, 28 Jun 2024 15:10:27 +0530 Subject: [PATCH 192/227] Address some more review comments --- beacon_node/execution_layer/src/engine_api.rs | 61 ++++++++++--------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index e0bfb7d49eb..6a56a5d076f 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -64,7 +64,6 @@ pub enum Error { ExecutionHeadBlockNotFound, ParentHashEqualsBlockHash(ExecutionBlockHash), PayloadIdUnavailable, - TransitionConfigurationMismatch, SszError(ssz_types::Error), DeserializeWithdrawals(ssz_types::Error), DeserializeDepositRequests(ssz_types::Error), @@ -635,38 +634,42 @@ impl ExecutionPayloadBodyV1 { } } ExecutionPayloadHeader::Electra(header) => { - let (Some(withdrawals), Some(deposit_requests), Some(withdrawal_requests)) = ( + let withdrawals_exist = self.withdrawals.is_some(); + let deposit_requests_exist = self.deposit_requests.is_some(); + let withdrawal_requests_exist = self.withdrawal_requests.is_some(); + if let (Some(withdrawals), Some(deposit_requests), Some(withdrawal_requests)) = ( self.withdrawals, self.deposit_requests, self.withdrawal_requests, - ) else { - return Err(format!( + ) { + Ok(ExecutionPayload::Electra(ExecutionPayloadElectra { + parent_hash: header.parent_hash, + fee_recipient: header.fee_recipient, + state_root: header.state_root, + receipts_root: header.receipts_root, + logs_bloom: header.logs_bloom, + prev_randao: header.prev_randao, + block_number: header.block_number, + gas_limit: header.gas_limit, + gas_used: header.gas_used, + timestamp: header.timestamp, + extra_data: header.extra_data, + base_fee_per_gas: header.base_fee_per_gas, + block_hash: header.block_hash, + transactions: self.transactions, + withdrawals, + blob_gas_used: header.blob_gas_used, + excess_blob_gas: header.excess_blob_gas, + deposit_requests, + withdrawal_requests, + })) + } else { + Err(format!( "block {} is post-electra but payload body doesn't have withdrawals/deposit_requests/withdrawal_requests \ - Check that ELs are returning receipts and withdrawal_requests in getPayloadBody requests", - header.block_hash - )); - }; - Ok(ExecutionPayload::Electra(ExecutionPayloadElectra { - parent_hash: header.parent_hash, - fee_recipient: header.fee_recipient, - state_root: header.state_root, - receipts_root: header.receipts_root, - logs_bloom: header.logs_bloom, - prev_randao: header.prev_randao, - block_number: header.block_number, - gas_limit: header.gas_limit, - gas_used: header.gas_used, - timestamp: header.timestamp, - extra_data: header.extra_data, - base_fee_per_gas: header.base_fee_per_gas, - block_hash: header.block_hash, - transactions: self.transactions, - withdrawals, - blob_gas_used: header.blob_gas_used, - excess_blob_gas: header.excess_blob_gas, - deposit_requests, - withdrawal_requests, - })) + withdrawals: {}, deposit_requests: {}, withdrawal_requests: {}", + header.block_hash, withdrawals_exist, deposit_requests_exist, withdrawal_requests_exist + )) + } } } } From aef797de788ff8a1dbbb16c2eb6115b5856bc887 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Fri, 28 Jun 2024 15:03:48 -0500 Subject: [PATCH 193/227] Electra: Get `devnet-1` ef-tests Working --- consensus/state_processing/src/genesis.rs | 11 ++++++ .../process_operations.rs | 7 +++- consensus/types/src/beacon_state.rs | 1 - testing/ef_tests/Makefile | 2 +- testing/ef_tests/src/cases/operations.rs | 34 ++++++++++++++++--- testing/ef_tests/tests/tests.rs | 5 +-- 6 files changed, 51 insertions(+), 9 deletions(-) diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 049599ea945..00697def5d2 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -7,6 +7,7 @@ use crate::upgrade::{ upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, }; use safe_arith::{ArithError, SafeArith}; +use std::sync::Arc; use tree_hash::TreeHash; use types::*; @@ -122,6 +123,16 @@ pub fn initialize_beacon_state_from_eth1( // Remove intermediate Deneb fork from `state.fork`. state.fork_mut().previous_version = spec.electra_fork_version; + // TODO(electra): think about this more and determine the best way to + // do this. The spec tests will expect that the sync committees are + // calculated using the electra value for MAX_EFFECTIVE_BALANCE when + // calling `initialize_beacon_state_from_eth1()`. But the sync committees + // are actually calcuated back in `upgrade_to_altair()`. We need to + // re-calculate the sync committees here now that the state is `Electra` + let sync_committee = Arc::new(state.get_next_sync_committee(spec)?); + *state.current_sync_committee_mut()? = sync_committee.clone(); + *state.next_sync_committee_mut()? = sync_committee; + // Override latest execution payload header. // See https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#testing if let Some(ExecutionPayloadHeader::Electra(header)) = execution_payload_header { diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index be5b60fe0b4..74166f67130 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -725,7 +725,12 @@ pub fn process_consolidation_request( } // Initiate source validator exit and append pending consolidation - initiate_validator_exit(state, source_index, spec)?; + let source_exit_epoch = state + .compute_consolidation_epoch_and_update_churn(source_validator.effective_balance, spec)?; + let source_validator = state.get_validator_mut(source_index)?; + source_validator.exit_epoch = source_exit_epoch; + source_validator.withdrawable_epoch = + source_exit_epoch.safe_add(spec.min_validator_withdrawability_delay)?; state .pending_consolidations_mut()? .push(PendingConsolidation { diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 8642036ad61..1fb8e15f82c 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -1090,7 +1090,6 @@ impl BeaconState { let active_validator_count = active_validator_indices.len(); let seed = self.get_seed(epoch, Domain::SyncCommittee, spec)?; - let max_effective_balance = spec.max_effective_balance_for_fork(self.fork_name_unchecked()); let mut i = 0; diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index 5dc3d2a0404..59e08e5d259 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,4 +1,4 @@ -TESTS_TAG := v1.5.0-alpha.2 +TESTS_TAG := v1.5.0-alpha.3 TESTS = general minimal mainnet TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 7cd22bba8f8..312ba474b62 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -7,7 +7,7 @@ use ssz::Decode; use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache; use state_processing::epoch_cache::initialize_epoch_cache; use state_processing::per_block_processing::process_operations::{ - process_deposit_requests, process_withdrawal_requests, + process_consolidation_requests, process_deposit_requests, process_withdrawal_requests, }; use state_processing::{ per_block_processing::{ @@ -25,8 +25,9 @@ use std::fmt::Debug; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockBodyBellatrix, BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconState, - BlindedPayload, Deposit, DepositRequest, ExecutionPayload, FullPayload, ProposerSlashing, - SignedBlsToExecutionChange, SignedVoluntaryExit, SyncAggregate, WithdrawalRequest, + BlindedPayload, ConsolidationRequest, Deposit, DepositRequest, ExecutionPayload, FullPayload, + ProposerSlashing, SignedBlsToExecutionChange, SignedVoluntaryExit, SyncAggregate, + WithdrawalRequest, }; #[derive(Debug, Clone, Default, Deserialize)] @@ -446,7 +447,7 @@ impl Operation for SignedBlsToExecutionChange { impl Operation for WithdrawalRequest { fn handler_name() -> String { - "execution_layer_withdrawal_request".into() + "withdrawal_request".into() } fn is_enabled_for_fork(fork_name: ForkName) -> bool { @@ -463,6 +464,7 @@ impl Operation for WithdrawalRequest { spec: &ChainSpec, _extra: &Operations, ) -> Result<(), BlockProcessingError> { + state.update_pubkey_cache()?; process_withdrawal_requests(state, &[self.clone()], spec) } } @@ -490,6 +492,30 @@ impl Operation for DepositRequest { } } +impl Operation for ConsolidationRequest { + fn handler_name() -> String { + "consolidation_request".into() + } + + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name >= ForkName::Electra + } + + fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { + ssz_decode_file(path) + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + _extra: &Operations, + ) -> Result<(), BlockProcessingError> { + state.update_pubkey_cache()?; + process_consolidation_requests(state, &[self.clone()], spec) + } +} + impl> LoadCase for Operations { fn load_from_dir(path: &Path, fork_name: ForkName) -> Result { let spec = &testing_spec::(fork_name); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 37699a48a2c..b93fce9e5e8 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -1,7 +1,7 @@ #![cfg(feature = "ef_tests")] use ef_tests::*; -use types::{MainnetEthSpec, MinimalEthSpec, WithdrawalRequest, *}; +use types::*; // Check that the hand-computed multiplications on EthSpec are correctly computed. // This test lives here because one is most likely to muck these up during a spec update. @@ -107,7 +107,8 @@ fn operations_deposit_requests() { #[test] fn operations_consolidations() { - todo!("implement once el triggered consolidations are good"); + OperationsHandler::::default().run(); + OperationsHandler::::default().run(); } #[test] From c9fe10b366ea4fe66037ee1c69f64f5ed6eb8319 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 1 Jul 2024 08:21:17 -0700 Subject: [PATCH 194/227] Update beacon_node/beacon_chain/src/electra_readiness.rs --- beacon_node/beacon_chain/src/electra_readiness.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/electra_readiness.rs b/beacon_node/beacon_chain/src/electra_readiness.rs index 669662cc945..551d43f9fd6 100644 --- a/beacon_node/beacon_chain/src/electra_readiness.rs +++ b/beacon_node/beacon_chain/src/electra_readiness.rs @@ -19,7 +19,7 @@ pub const ENGINE_CAPABILITIES_REFRESH_INTERVAL: u64 = 300; pub enum ElectraReadiness { /// The execution engine is electra-enabled (as far as we can tell) Ready, - /// We are connected to an execution engine which doesn't support the V3 engine api methods + /// We are connected to an execution engine which doesn't support the V4 engine api methods V4MethodsNotSupported { error: String }, /// The transition configuration with the EL failed, there might be a problem with /// connectivity, authentication or a difference in configuration. From 69ac34209cd26bed60bd5254f9aa6a73ceaa5c0d Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 1 Jul 2024 08:21:31 -0700 Subject: [PATCH 195/227] Update consensus/types/src/chain_spec.rs --- consensus/types/src/chain_spec.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 87e3fcdd0f8..16d4f046ba6 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -180,7 +180,6 @@ pub struct ChainSpec { pub electra_fork_version: [u8; 4], /// The Electra fork epoch is optional, with `None` representing "Electra never happens". pub electra_fork_epoch: Option, - // TODO(pawan): this shouldn't be a config parameter? pub unset_deposit_requests_start_index: u64, pub full_exit_request_amount: u64, pub min_activation_balance: u64, From ce66f5312cf7c95c7ab2c186ef70a7a0aab5a673 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 3 Jul 2024 10:09:10 +1000 Subject: [PATCH 196/227] Update pending balance deposit processing (#6005) * Update pending balance deposit processing * Update single_pass.rs --------- Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> --- .../src/per_epoch_processing/single_pass.rs | 61 ++++++++++++++++--- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/consensus/state_processing/src/per_epoch_processing/single_pass.rs b/consensus/state_processing/src/per_epoch_processing/single_pass.rs index 4ab756b561b..402c721b592 100644 --- a/consensus/state_processing/src/per_epoch_processing/single_pass.rs +++ b/consensus/state_processing/src/per_epoch_processing/single_pass.rs @@ -17,8 +17,8 @@ use types::{ }, milhouse::Cow, ActivationQueue, BeaconState, BeaconStateError, ChainSpec, Checkpoint, Epoch, EthSpec, - ExitCache, ForkName, List, ParticipationFlags, ProgressiveBalancesCache, RelativeEpoch, - Unsigned, Validator, + ExitCache, ForkName, List, ParticipationFlags, PendingBalanceDeposit, ProgressiveBalancesCache, + RelativeEpoch, Unsigned, Validator, }; pub struct SinglePassConfig { @@ -91,6 +91,8 @@ struct PendingBalanceDepositsContext { deposit_balance_to_consume: u64, /// Total balance increases for each validator due to pending balance deposits. validator_deposits_to_process: HashMap, + /// The deposits to append to `pending_balance_deposits` after processing all applicable deposits. + deposits_to_postpone: Vec, } struct EffectiveBalancesContext { @@ -342,12 +344,15 @@ pub fn process_epoch_single_pass( // of the `pending_balance_deposits` list. But we may as well preserve the write ordering used // by the spec and do this first. if let Some(ctxt) = pending_balance_deposits_ctxt { - let new_pending_balance_deposits = List::try_from_iter( + let mut new_pending_balance_deposits = List::try_from_iter( state .pending_balance_deposits()? .iter_from(ctxt.next_deposit_index)? .cloned(), )?; + for deposit in ctxt.deposits_to_postpone { + new_pending_balance_deposits.push(deposit)?; + } *state.pending_balance_deposits_mut()? = new_pending_balance_deposits; *state.deposit_balance_to_consume_mut()? = ctxt.deposit_balance_to_consume; } @@ -805,22 +810,57 @@ impl PendingBalanceDepositsContext { let available_for_processing = state .deposit_balance_to_consume()? .safe_add(state.get_activation_exit_churn_limit(spec)?)?; + let current_epoch = state.current_epoch(); let mut processed_amount = 0; let mut next_deposit_index = 0; let mut validator_deposits_to_process = HashMap::new(); + let mut deposits_to_postpone = vec![]; let pending_balance_deposits = state.pending_balance_deposits()?; for deposit in pending_balance_deposits.iter() { - if processed_amount.safe_add(deposit.amount)? > available_for_processing { - break; + // We have to do a bit of indexing into `validators` here, but I can't see any way + // around that without changing the spec. + // + // We need to work out if `validator.exit_epoch` will be set to a non-default value + // *after* changes applied by `process_registry_updates`, which in our implementation + // does not happen until after this (but in the spec happens before). However it's not + // hard to work out: we don't need to know exactly what value the `exit_epoch` will + // take, just whether it is non-default. Nor do we need to know the value of + // `withdrawable_epoch`, because `current_epoch <= withdrawable_epoch` will evaluate to + // `true` both for the actual value & the default placeholder value (`FAR_FUTURE_EPOCH`). + let validator = state.get_validator(deposit.index as usize)?; + let already_exited = validator.exit_epoch < spec.far_future_epoch; + // In the spec process_registry_updates is called before process_pending_balance_deposits + // so we must account for process_registry_updates ejecting the validator for low balance + // and setting the exit_epoch to < far_future_epoch + let will_be_exited = validator.is_active_at(current_epoch) + && validator.effective_balance <= spec.ejection_balance; + if already_exited || will_be_exited { + if state.current_epoch() <= validator.withdrawable_epoch { + deposits_to_postpone.push(deposit.clone()); + } else { + // Deposited balance will never become active. Increase balance but do not + // consume churn. + validator_deposits_to_process + .entry(deposit.index as usize) + .or_insert(0) + .safe_add_assign(deposit.amount)?; + } + } else { + // Deposit does not fit in the churn, no more deposit processing in this epoch. + if processed_amount.safe_add(deposit.amount)? > available_for_processing { + break; + } + // Deposit fits in the churn, process it. Increase balance and consume churn. + validator_deposits_to_process + .entry(deposit.index as usize) + .or_insert(0) + .safe_add_assign(deposit.amount)?; + processed_amount.safe_add_assign(deposit.amount)?; } - validator_deposits_to_process - .entry(deposit.index as usize) - .or_insert(0) - .safe_add_assign(deposit.amount)?; - processed_amount.safe_add_assign(deposit.amount)?; + // Regardless of how the deposit was handled, we move on in the queue. next_deposit_index.safe_add_assign(1)?; } @@ -834,6 +874,7 @@ impl PendingBalanceDepositsContext { next_deposit_index, deposit_balance_to_consume, validator_deposits_to_process, + deposits_to_postpone, }) } } From fe5730c58fb8b2a0ee2e40f2f2a41dd529ea7a5d Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Thu, 4 Jul 2024 10:27:12 -0500 Subject: [PATCH 197/227] Fix Bug in `engine_api` by Renaming Field --- .../execution_layer/src/engine_api/json_structures.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 0f566c8061e..aed3da7bf10 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -903,7 +903,7 @@ impl From for DepositRequest { #[serde(rename_all = "camelCase")] pub struct JsonWithdrawalRequest { pub source_address: Address, - pub validator_public_key: PublicKeyBytes, + pub validator_pub_key: PublicKeyBytes, #[serde(with = "serde_utils::u64_hex_be")] pub amount: u64, } @@ -912,7 +912,7 @@ impl From for JsonWithdrawalRequest { fn from(withdrawal_request: WithdrawalRequest) -> Self { Self { source_address: withdrawal_request.source_address, - validator_public_key: withdrawal_request.validator_pubkey, + validator_pub_key: withdrawal_request.validator_pubkey, amount: withdrawal_request.amount, } } @@ -922,7 +922,7 @@ impl From for WithdrawalRequest { fn from(json_withdrawal_request: JsonWithdrawalRequest) -> Self { Self { source_address: json_withdrawal_request.source_address, - validator_pubkey: json_withdrawal_request.validator_public_key, + validator_pubkey: json_withdrawal_request.validator_pub_key, amount: json_withdrawal_request.amount, } } From 60359d6935ae7943eca9eb7624bf9385306693e0 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Thu, 4 Jul 2024 11:24:35 -0500 Subject: [PATCH 198/227] one more stupid rename --- .../execution_layer/src/engine_api/json_structures.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index aed3da7bf10..bfac5c87dd9 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -903,7 +903,7 @@ impl From for DepositRequest { #[serde(rename_all = "camelCase")] pub struct JsonWithdrawalRequest { pub source_address: Address, - pub validator_pub_key: PublicKeyBytes, + pub validator_pubkey: PublicKeyBytes, #[serde(with = "serde_utils::u64_hex_be")] pub amount: u64, } @@ -912,7 +912,7 @@ impl From for JsonWithdrawalRequest { fn from(withdrawal_request: WithdrawalRequest) -> Self { Self { source_address: withdrawal_request.source_address, - validator_pub_key: withdrawal_request.validator_pubkey, + validator_pubkey: withdrawal_request.validator_pubkey, amount: withdrawal_request.amount, } } @@ -922,7 +922,7 @@ impl From for WithdrawalRequest { fn from(json_withdrawal_request: JsonWithdrawalRequest) -> Self { Self { source_address: json_withdrawal_request.source_address, - validator_pubkey: json_withdrawal_request.validator_pub_key, + validator_pubkey: json_withdrawal_request.validator_pubkey, amount: json_withdrawal_request.amount, } } From 6766f329e3998de7d5249dcd2966a41f03b9b82e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 8 Jul 2024 14:04:36 -0700 Subject: [PATCH 199/227] update GET requests --- beacon_node/http_api/src/lib.rs | 109 ++++++++++++++++++++++++-------- 1 file changed, 82 insertions(+), 27 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 7db0889efd9..15a045e26c0 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -31,6 +31,7 @@ mod validators; mod version; use crate::produce_block::{produce_blinded_block_v2, produce_block_v2, produce_block_v3}; +use crate::version::fork_versioned_response; use beacon_chain::{ attestation_verification::VerifiedAttestation, observed_operations::ObservationOutcome, validator_monitor::timestamp_now, AttestationError as AttnError, BeaconChain, BeaconChainError, @@ -258,11 +259,13 @@ pub fn prometheus_metrics() -> warp::filters::log::Log( ); // GET beacon/blocks/{block_id}/attestations - let get_beacon_block_attestations = beacon_blocks_path_v1 + let get_beacon_block_attestations = beacon_blocks_path_any .clone() .and(warp::path("attestations")) .and(warp::path::end()) .then( - |block_id: BlockId, + |endpoint_version: EndpointVersion, + block_id: BlockId, task_spawner: TaskSpawner, chain: Arc>| { - task_spawner.blocking_json_task(Priority::P1, move || { + task_spawner.blocking_response_task(Priority::P1, move || { let (block, execution_optimistic, finalized) = block_id.blinded_block(&chain)?; - Ok(api_types::GenericResponse::from( - block - .message() - .body() - .attestations() - .map(|att| att.clone_as_attestation()) - .collect::>(), - ) - .add_execution_optimistic_finalized(execution_optimistic, finalized)) + let fork_name = block + .fork_name(&chain.spec) + .map_err(inconsistent_fork_rejection)?; + let atts = block + .message() + .body() + .attestations() + .map(|att| att.clone_as_attestation()) + .collect::>(); + let res = execution_optimistic_finalized_fork_versioned_response( + endpoint_version, + fork_name, + execution_optimistic, + finalized, + &atts, + )?; + Ok(add_consensus_version_header( + warp::reply::json(&res).into_response(), + fork_name, + )) }) }, ); @@ -1751,8 +1766,14 @@ pub fn serve( .and(task_spawner_filter.clone()) .and(chain_filter.clone()); + let beacon_pool_path_any = any_version + .and(warp::path("beacon")) + .and(warp::path("pool")) + .and(task_spawner_filter.clone()) + .and(chain_filter.clone()); + // POST beacon/pool/attestations - let post_beacon_pool_attestations = beacon_pool_path + let post_beacon_pool_attestations = beacon_pool_path_any .clone() .and(warp::path("attestations")) .and(warp::path::end()) @@ -1761,7 +1782,8 @@ pub fn serve( .and(reprocess_send_filter) .and(log_filter.clone()) .then( - |task_spawner: TaskSpawner, + |endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, chain: Arc>, attestations: Vec>, network_tx: UnboundedSender>, @@ -1782,16 +1804,17 @@ pub fn serve( ); // GET beacon/pool/attestations?committee_index,slot - let get_beacon_pool_attestations = beacon_pool_path + let get_beacon_pool_attestations = beacon_pool_path_any .clone() .and(warp::path("attestations")) .and(warp::path::end()) .and(warp::query::()) .then( - |task_spawner: TaskSpawner, + |endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, chain: Arc>, query: api_types::AttestationPoolQuery| { - task_spawner.blocking_json_task(Priority::P1, move || { + task_spawner.blocking_response_task(Priority::P1, move || { let query_filter = |data: &AttestationData| { query.slot.map_or(true, |slot| slot == data.slot) && query @@ -1808,20 +1831,37 @@ pub fn serve( .filter(|&att| query_filter(att.data())) .cloned(), ); - Ok(api_types::GenericResponse::from(attestations)) + let slot = query + .slot + .or_else(|| { + attestations + .first() + .map(|att| att.data().slot) + .or_else(|| chain.slot_clock.now()) + }) + .ok_or(warp_utils::reject::custom_server_error( + "unable to read slot clock".to_string(), + ))?; + let fork_name = chain.spec.fork_name_at_slot::(slot); + let res = fork_versioned_response(endpoint_version, fork_name, &attestations)?; + Ok(add_consensus_version_header( + warp::reply::json(&res).into_response(), + fork_name, + )) }) }, ); // POST beacon/pool/attester_slashings - let post_beacon_pool_attester_slashings = beacon_pool_path + let post_beacon_pool_attester_slashings = beacon_pool_path_any .clone() .and(warp::path("attester_slashings")) .and(warp::path::end()) .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .then( - |task_spawner: TaskSpawner, + |endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, chain: Arc>, slashing: AttesterSlashing, network_tx: UnboundedSender>| { @@ -1858,15 +1898,29 @@ pub fn serve( ); // GET beacon/pool/attester_slashings - let get_beacon_pool_attester_slashings = beacon_pool_path + let get_beacon_pool_attester_slashings = beacon_pool_path_any .clone() .and(warp::path("attester_slashings")) .and(warp::path::end()) .then( - |task_spawner: TaskSpawner, chain: Arc>| { - task_spawner.blocking_json_task(Priority::P1, move || { - let attestations = chain.op_pool.get_all_attester_slashings(); - Ok(api_types::GenericResponse::from(attestations)) + |endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_response_task(Priority::P1, move || { + let slashings = chain.op_pool.get_all_attester_slashings(); + let slot = slashings + .first() + .map(|slashing| slashing.attestation_1().data().slot) + .or_else(|| chain.slot_clock.now()) + .ok_or(warp_utils::reject::custom_server_error( + "unable to read slot clock".to_string(), + ))?; + let fork_name = chain.spec.fork_name_at_slot::(slot); + let res = fork_versioned_response(endpoint_version, fork_name, &slashings)?; + Ok(add_consensus_version_header( + warp::reply::json(&res).into_response(), + fork_name, + )) }) }, ); @@ -3319,7 +3373,7 @@ pub fn serve( ); // POST validator/aggregate_and_proofs - let post_validator_aggregate_and_proofs = eth_v1 + let post_validator_aggregate_and_proofs = any_version .and(warp::path("validator")) .and(warp::path("aggregate_and_proofs")) .and(warp::path::end()) @@ -3330,7 +3384,8 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |not_synced_filter: Result<(), Rejection>, + |endpoint_version: EndpointVersion, + not_synced_filter: Result<(), Rejection>, task_spawner: TaskSpawner, chain: Arc>, aggregates: Vec>, From 39d41ada934bd7f6feea647bcb688e923128ec29 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 8 Jul 2024 14:14:20 -0700 Subject: [PATCH 200/227] update POST requests --- beacon_node/http_api/src/lib.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 15a045e26c0..6656485dceb 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1782,7 +1782,10 @@ pub fn serve( .and(reprocess_send_filter) .and(log_filter.clone()) .then( - |endpoint_version: EndpointVersion, + // V1 and V2 are identical except V2 has a consensus version header in the request. + // We only require this header for SSZ deserialization, which isn't supported for + // this endpoint presently. + |_endpoint_version: EndpointVersion, task_spawner: TaskSpawner, chain: Arc>, attestations: Vec>, @@ -1860,7 +1863,10 @@ pub fn serve( .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .then( - |endpoint_version: EndpointVersion, + // V1 and V2 are identical except V2 has a consensus version header in the request. + // We only require this header for SSZ deserialization, which isn't supported for + // this endpoint presently. + |_endpoint_version: EndpointVersion, task_spawner: TaskSpawner, chain: Arc>, slashing: AttesterSlashing, @@ -3384,7 +3390,10 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |endpoint_version: EndpointVersion, + // V1 and V2 are identical except V2 has a consensus version header in the request. + // We only require this header for SSZ deserialization, which isn't supported for + // this endpoint presently. + |_endpoint_version: EndpointVersion, not_synced_filter: Result<(), Rejection>, task_spawner: TaskSpawner, chain: Arc>, From f405601d6f898e4693353ef9e72881da5eb27b58 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 8 Jul 2024 18:19:10 -0700 Subject: [PATCH 201/227] add client updates and test updates --- beacon_node/http_api/tests/fork_tests.rs | 13 +- .../http_api/tests/interactive_tests.rs | 3 +- beacon_node/http_api/tests/tests.rs | 136 ++++++++++-- common/eth2/src/lib.rs | 197 ++++++++++++++++-- consensus/types/src/attestation.rs | 25 ++- consensus/types/src/attester_slashing.rs | 23 ++ validator_client/src/attestation_service.rs | 38 +++- 7 files changed, 397 insertions(+), 38 deletions(-) diff --git a/beacon_node/http_api/tests/fork_tests.rs b/beacon_node/http_api/tests/fork_tests.rs index ad32ff1d579..08dd7f06f70 100644 --- a/beacon_node/http_api/tests/fork_tests.rs +++ b/beacon_node/http_api/tests/fork_tests.rs @@ -150,8 +150,13 @@ async fn attestations_across_fork_with_skip_slots() { .collect::>(); assert!(!unaggregated_attestations.is_empty()); + let fork_name = harness.spec.fork_name_at_slot::(fork_slot); client - .post_beacon_pool_attestations(&unaggregated_attestations) + .post_beacon_pool_attestations_v1(&unaggregated_attestations) + .await + .unwrap(); + client + .post_beacon_pool_attestations_v2(&unaggregated_attestations, fork_name) .await .unwrap(); @@ -162,7 +167,11 @@ async fn attestations_across_fork_with_skip_slots() { assert!(!signed_aggregates.is_empty()); client - .post_validator_aggregate_and_proof(&signed_aggregates) + .post_validator_aggregate_and_proof_v1(&signed_aggregates) + .await + .unwrap(); + client + .post_validator_aggregate_and_proof_v2(&signed_aggregates, fork_name) .await .unwrap(); } diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 711820ccac6..a2c48b98c91 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -890,9 +890,10 @@ async fn queue_attestations_from_http() { .flat_map(|attestations| attestations.into_iter().map(|(att, _subnet)| att)) .collect::>(); + let fork_name = tester.harness.spec.fork_name_at_slot::(attestation_slot); let attestation_future = tokio::spawn(async move { client - .post_beacon_pool_attestations(&attestations) + .post_beacon_pool_attestations_v2(&attestations, fork_name) .await .expect("attestations should be processed successfully") }); diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 5baf96091bf..6068b4a0937 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -1633,7 +1633,7 @@ impl ApiTester { for block_id in self.interesting_block_ids() { let result = self .client - .get_beacon_blocks_attestations(block_id.0) + .get_beacon_blocks_attestations_v2(block_id.0) .await .unwrap() .map(|res| res.data); @@ -1666,7 +1666,7 @@ impl ApiTester { pub async fn test_post_beacon_pool_attestations_valid(mut self) -> Self { self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -1675,6 +1675,20 @@ impl ApiTester { "valid attestation should be sent to network" ); + let fork_name = self + .attestations + .first() + .map(|att| self.chain.spec.fork_name_at_slot::(att.data().slot)) + .unwrap(); + self.client + .post_beacon_pool_attestations_v2(self.attestations.as_slice(), fork_name) + .await + .unwrap(); + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "valid attestation should be sent to network" + ); + self } @@ -1691,7 +1705,7 @@ impl ApiTester { let err = self .client - .post_beacon_pool_attestations(attestations.as_slice()) + .post_beacon_pool_attestations_v1(attestations.as_slice()) .await .unwrap_err(); @@ -1712,6 +1726,35 @@ impl ApiTester { "if some attestations are valid, we should send them to the network" ); + let fork_name = self + .attestations + .first() + .map(|att| self.chain.spec.fork_name_at_slot::(att.data().slot)) + .unwrap(); + + let err_v2 = self + .client + .post_beacon_pool_attestations_v2(attestations.as_slice(), fork_name) + .await + .unwrap_err(); + + match err_v2 { + Error::ServerIndexedMessage(IndexedErrorMessage { + code, + message: _, + failures, + }) => { + assert_eq!(code, 400); + assert_eq!(failures.len(), self.attestations.len()); + } + _ => panic!("query did not fail correctly"), + } + + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "if some attestations are valid, we should send them to the network" + ); + self } @@ -1777,7 +1820,7 @@ impl ApiTester { pub async fn test_get_beacon_pool_attestations(self) -> Self { let result = self .client - .get_beacon_pool_attestations(None, None) + .get_beacon_pool_attestations_v1(None, None) .await .unwrap() .data; @@ -1787,12 +1830,34 @@ impl ApiTester { assert_eq!(result, expected); + let result = self + .client + .get_beacon_pool_attestations_v2(None, None) + .await + .unwrap() + .data; + assert_eq!(result, expected); + self } pub async fn test_post_beacon_pool_attester_slashings_valid(mut self) -> Self { self.client - .post_beacon_pool_attester_slashings(&self.attester_slashing) + .post_beacon_pool_attester_slashings_v1(&self.attester_slashing) + .await + .unwrap(); + + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "valid attester slashing should be sent to network" + ); + + let fork_name = self + .chain + .spec + .fork_name_at_slot::(self.attester_slashing.attestation_1().data().slot); + self.client + .post_beacon_pool_attester_slashings_v2(&self.attester_slashing, fork_name) .await .unwrap(); @@ -1816,7 +1881,21 @@ impl ApiTester { } self.client - .post_beacon_pool_attester_slashings(&slashing) + .post_beacon_pool_attester_slashings_v1(&slashing) + .await + .unwrap_err(); + + assert!( + self.network_rx.network_recv.recv().now_or_never().is_none(), + "invalid attester slashing should not be sent to network" + ); + + let fork_name = self + .chain + .spec + .fork_name_at_slot::(self.attester_slashing.attestation_1().data().slot); + self.client + .post_beacon_pool_attester_slashings_v2(&slashing, fork_name) .await .unwrap_err(); @@ -1831,7 +1910,7 @@ impl ApiTester { pub async fn test_get_beacon_pool_attester_slashings(self) -> Self { let result = self .client - .get_beacon_pool_attester_slashings() + .get_beacon_pool_attester_slashings_v1() .await .unwrap() .data; @@ -1840,6 +1919,14 @@ impl ApiTester { assert_eq!(result, expected); + let result = self + .client + .get_beacon_pool_attester_slashings_v2() + .await + .unwrap() + .data; + assert_eq!(result, expected); + self } @@ -3210,7 +3297,7 @@ impl ApiTester { .get_validator_aggregate_attestation_v2( attestation.data().slot, attestation.data().tree_hash_root(), - attestation.committee_index(), + attestation.committee_index().unwrap(), ) .await .unwrap() @@ -3346,7 +3433,18 @@ impl ApiTester { let aggregate = self.get_aggregate().await; self.client - .post_validator_aggregate_and_proof::(&[aggregate]) + .post_validator_aggregate_and_proof_v1::(&[aggregate.clone()]) + .await + .unwrap(); + + assert!(self.network_rx.network_recv.recv().await.is_some()); + + let fork_name = self + .chain + .spec + .fork_name_at_slot::(aggregate.message().aggregate().data().slot); + self.client + .post_validator_aggregate_and_proof_v2::(&[aggregate], fork_name) .await .unwrap(); @@ -3367,12 +3465,22 @@ impl ApiTester { } self.client - .post_validator_aggregate_and_proof::(&[aggregate]) + .post_validator_aggregate_and_proof_v1::(&[aggregate.clone()]) .await .unwrap_err(); assert!(self.network_rx.network_recv.recv().now_or_never().is_none()); + let fork_name = self + .chain + .spec + .fork_name_at_slot::(aggregate.message().aggregate().data().slot); + self.client + .post_validator_aggregate_and_proof_v2::(&[aggregate], fork_name) + .await + .unwrap(); + assert!(self.network_rx.network_recv.recv().now_or_never().is_none()); + self } @@ -3471,7 +3579,7 @@ impl ApiTester { pub async fn test_post_validator_register_validator_slashed(self) -> Self { // slash a validator self.client - .post_beacon_pool_attester_slashings(&self.attester_slashing) + .post_beacon_pool_attester_slashings_v1(&self.attester_slashing) .await .unwrap(); @@ -3584,7 +3692,7 @@ impl ApiTester { // Attest to the current slot self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -5224,7 +5332,7 @@ impl ApiTester { // Attest to the current slot self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -5279,7 +5387,7 @@ impl ApiTester { let expected_attestation_len = self.attestations.len(); self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 532429df3ba..ff85d6e64f8 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -346,6 +346,19 @@ impl BeaconNodeHttpClient { Ok(()) } + /// Perform a HTTP POST request with a custom timeout. + async fn post_with_timeout_and_consensus_header( + &self, + url: U, + body: &T, + timeout: Duration, + fork_name: ForkName, + ) -> Result<(), Error> { + self.post_generic_with_consensus_version(url, body, Some(timeout), fork_name) + .await?; + Ok(()) + } + /// Perform a HTTP POST request with a custom timeout, returning a JSON response. async fn post_with_timeout_and_response( &self, @@ -395,6 +408,33 @@ impl BeaconNodeHttpClient { ok_or_error(response).await } + /// Generic POST function supporting arbitrary responses and timeouts. + /// Does not include Content-Type application/json in the request header. + async fn post_generic_json_without_content_type_header_but_with_consensus_header< + T: Serialize, + U: IntoUrl, + >( + &self, + url: U, + body: &T, + timeout: Option, + fork: ForkName, + ) -> Result { + let mut builder = self.client.post(url); + if let Some(timeout) = timeout { + builder = builder.timeout(timeout); + } + + let serialized_body = serde_json::to_vec(body).map_err(Error::InvalidJson)?; + + let response = builder + .header(CONSENSUS_VERSION_HEADER, fork.to_string()) + .body(serialized_body) + .send() + .await?; + ok_or_error(response).await + } + /// Generic POST function supporting arbitrary responses and timeouts. async fn post_generic_with_consensus_version( &self, @@ -1191,10 +1231,10 @@ impl BeaconNodeHttpClient { self.get_opt(path).await } - /// `GET beacon/blocks/{block_id}/attestations` + /// `GET v1/beacon/blocks/{block_id}/attestations` /// /// Returns `Ok(None)` on a 404 error. - pub async fn get_beacon_blocks_attestations( + pub async fn get_beacon_blocks_attestations_v1( &self, block_id: BlockId, ) -> Result>>>, Error> { @@ -1210,8 +1250,28 @@ impl BeaconNodeHttpClient { self.get_opt(path).await } - /// `POST beacon/pool/attestations` - pub async fn post_beacon_pool_attestations( + /// `GET v2/beacon/blocks/{block_id}/attestations` + /// + /// Returns `Ok(None)` on a 404 error. + pub async fn get_beacon_blocks_attestations_v2( + &self, + block_id: BlockId, + ) -> Result>>>, Error> + { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("blocks") + .push(&block_id.to_string()) + .push("attestations"); + + self.get_opt(path).await + } + + /// `POST v1/beacon/pool/attestations` + pub async fn post_beacon_pool_attestations_v1( &self, attestations: &[Attestation], ) -> Result<(), Error> { @@ -1229,8 +1289,33 @@ impl BeaconNodeHttpClient { Ok(()) } - /// `GET beacon/pool/attestations?slot,committee_index` - pub async fn get_beacon_pool_attestations( + /// `POST v2/beacon/pool/attestations` + pub async fn post_beacon_pool_attestations_v2( + &self, + attestations: &[Attestation], + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attestations"); + + self.post_with_timeout_and_consensus_header( + path, + &attestations, + self.timeouts.attestation, + fork_name, + ) + .await?; + + Ok(()) + } + + /// `GET v1/beacon/pool/attestations?slot,committee_index` + pub async fn get_beacon_pool_attestations_v1( &self, slot: Option, committee_index: Option, @@ -1256,8 +1341,35 @@ impl BeaconNodeHttpClient { self.get(path).await } - /// `POST beacon/pool/attester_slashings` - pub async fn post_beacon_pool_attester_slashings( + /// `GET v2/beacon/pool/attestations?slot,committee_index` + pub async fn get_beacon_pool_attestations_v2( + &self, + slot: Option, + committee_index: Option, + ) -> Result>>, Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attestations"); + + if let Some(slot) = slot { + path.query_pairs_mut() + .append_pair("slot", &slot.to_string()); + } + + if let Some(index) = committee_index { + path.query_pairs_mut() + .append_pair("committee_index", &index.to_string()); + } + + self.get(path).await + } + + /// `POST v1/beacon/pool/attester_slashings` + pub async fn post_beacon_pool_attester_slashings_v1( &self, slashing: &AttesterSlashing, ) -> Result<(), Error> { @@ -1275,8 +1387,30 @@ impl BeaconNodeHttpClient { Ok(()) } - /// `GET beacon/pool/attester_slashings` - pub async fn get_beacon_pool_attester_slashings( + /// `POST v2/beacon/pool/attester_slashings` + pub async fn post_beacon_pool_attester_slashings_v2( + &self, + slashing: &AttesterSlashing, + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attester_slashings"); + + self.post_generic_json_without_content_type_header_but_with_consensus_header( + path, slashing, None, fork_name, + ) + .await?; + + Ok(()) + } + + /// `GET v1/beacon/pool/attester_slashings` + pub async fn get_beacon_pool_attester_slashings_v1( &self, ) -> Result>>, Error> { let mut path = self.eth_path(V1)?; @@ -1290,6 +1424,21 @@ impl BeaconNodeHttpClient { self.get(path).await } + /// `GET v2/beacon/pool/attester_slashings` + pub async fn get_beacon_pool_attester_slashings_v2( + &self, + ) -> Result>>, Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attester_slashings"); + + self.get(path).await + } + /// `POST beacon/pool/proposer_slashings` pub async fn post_beacon_pool_proposer_slashings( &self, @@ -2324,8 +2473,8 @@ impl BeaconNodeHttpClient { .await } - /// `POST validator/aggregate_and_proofs` - pub async fn post_validator_aggregate_and_proof( + /// `POST v1/validator/aggregate_and_proofs` + pub async fn post_validator_aggregate_and_proof_v1( &self, aggregates: &[SignedAggregateAndProof], ) -> Result<(), Error> { @@ -2342,6 +2491,30 @@ impl BeaconNodeHttpClient { Ok(()) } + /// `POST v2/validator/aggregate_and_proofs` + pub async fn post_validator_aggregate_and_proof_v2( + &self, + aggregates: &[SignedAggregateAndProof], + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("validator") + .push("aggregate_and_proofs"); + + self.post_with_timeout_and_consensus_header( + path, + &aggregates, + self.timeouts.attestation, + fork_name, + ) + .await?; + + Ok(()) + } + /// `POST validator/beacon_committee_subscriptions` pub async fn post_validator_beacon_committee_subscriptions( &self, diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 88993267a94..1acaee9654c 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -1,6 +1,6 @@ use crate::slot_data::SlotData; -use crate::Checkpoint; use crate::{test_utils::TestRandom, Hash256, Slot}; +use crate::{Checkpoint, ForkVersionDeserialize}; use derivative::Derivative; use safe_arith::ArithError; use serde::{Deserialize, Serialize}; @@ -487,6 +487,29 @@ impl<'a, E: EthSpec> From> for AttestationRef<'a, E> } } +impl ForkVersionDeserialize for Vec> { + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::Value, + fork_name: crate::ForkName, + ) -> Result { + if fork_name.electra_enabled() { + let attestations: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(attestations + .into_iter() + .map(Attestation::Electra) + .collect::>()) + } else { + let attestations: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(attestations + .into_iter() + .map(Attestation::Base) + .collect::>()) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/attester_slashing.rs b/consensus/types/src/attester_slashing.rs index a8d4e6989c8..feb10506ef0 100644 --- a/consensus/types/src/attester_slashing.rs +++ b/consensus/types/src/attester_slashing.rs @@ -160,6 +160,29 @@ impl AttesterSlashing { } } +impl crate::ForkVersionDeserialize for Vec> { + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::Value, + fork_name: crate::ForkName, + ) -> Result { + if fork_name.electra_enabled() { + let slashings: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(slashings + .into_iter() + .map(AttesterSlashing::Electra) + .collect::>()) + } else { + let slashings: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(slashings + .into_iter() + .map(AttesterSlashing::Base) + .collect::>()) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index db32dbcf4a5..5167f8b6499 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -14,7 +14,7 @@ use std::ops::Deref; use std::sync::Arc; use tokio::time::{sleep, sleep_until, Duration, Instant}; use tree_hash::TreeHash; -use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, ForkName, Slot}; +use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot}; /// Builds an `AttestationService`. pub struct AttestationServiceBuilder { @@ -449,6 +449,11 @@ impl AttestationService { warn!(log, "No attestations were published"); return Ok(None); } + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); // Post the attestations to the BN. match self @@ -462,9 +467,15 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::ATTESTATIONS_HTTP_POST], ); - beacon_node - .post_beacon_pool_attestations(attestations) - .await + if fork_name.electra_enabled() { + beacon_node + .post_beacon_pool_attestations_v2(attestations, fork_name) + .await + } else { + beacon_node + .post_beacon_pool_attestations_v1(attestations) + .await + } }, ) .await @@ -537,7 +548,7 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::AGGREGATES_HTTP_GET], ); - let aggregate_attestation_result = if fork_name >= ForkName::Electra { + let aggregate_attestation_result = if fork_name.electra_enabled() { beacon_node .get_validator_aggregate_attestation_v2( attestation_data.slot, @@ -627,9 +638,20 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::AGGREGATES_HTTP_POST], ); - beacon_node - .post_validator_aggregate_and_proof(signed_aggregate_and_proofs_slice) - .await + if fork_name.electra_enabled() { + beacon_node + .post_validator_aggregate_and_proof_v2( + signed_aggregate_and_proofs_slice, + fork_name, + ) + .await + } else { + beacon_node + .post_validator_aggregate_and_proof_v1( + signed_aggregate_and_proofs_slice, + ) + .await + } }, ) .await From 80266a81096ef600327a42dff82c0c20f1b76b57 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 8 Jul 2024 18:28:02 -0700 Subject: [PATCH 202/227] compile after merge --- beacon_node/beacon_chain/src/eth1_chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index 6e80abffc71..62aad558eeb 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -548,7 +548,7 @@ impl Eth1ChainBackend for CachingEth1Backend { // [New in Electra:EIP6110] let deposit_index_limit = - if let Ok(deposit_receipts_start_index) = state.deposit_receipts_start_index() { + if let Ok(deposit_receipts_start_index) = state.deposit_requests_start_index() { std::cmp::min(deposit_count, deposit_receipts_start_index) } else { deposit_count From d1357e459ac1f8e82463ba1b1a1cf5a1ce291932 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 8 Jul 2024 18:37:38 -0700 Subject: [PATCH 203/227] unwrap -> unwrap_err --- beacon_node/http_api/tests/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 6068b4a0937..53abca67ad6 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -3478,7 +3478,7 @@ impl ApiTester { self.client .post_validator_aggregate_and_proof_v2::(&[aggregate], fork_name) .await - .unwrap(); + .unwrap_err(); assert!(self.network_rx.network_recv.recv().now_or_never().is_none()); self From d3947462482638649b0e40ad75657262d16e1b12 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 8 Jul 2024 18:50:14 -0700 Subject: [PATCH 204/227] self review --- beacon_node/http_api/tests/tests.rs | 2 +- common/eth2/src/lib.rs | 5 +++-- consensus/types/src/attestation.rs | 17 ++++++++++++++ validator_client/src/attestation_service.rs | 25 +++++++++++++-------- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 53abca67ad6..8eabdeb36a9 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -3289,7 +3289,7 @@ impl ApiTester { .chain .spec .fork_name_at_slot::(self.chain.slot().unwrap()) - >= ForkName::Electra + .electra_enabled() { for attestation in self.chain.naive_aggregation_pool.read().iter() { let result = self diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index ff85d6e64f8..e6fa6eb02f3 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -346,7 +346,7 @@ impl BeaconNodeHttpClient { Ok(()) } - /// Perform a HTTP POST request with a custom timeout. + /// Perform a HTTP POST request with a custom timeout and consensus header. async fn post_with_timeout_and_consensus_header( &self, url: U, @@ -410,6 +410,7 @@ impl BeaconNodeHttpClient { /// Generic POST function supporting arbitrary responses and timeouts. /// Does not include Content-Type application/json in the request header. + /// Does include Eth-Consensus-Version in the request header. async fn post_generic_json_without_content_type_header_but_with_consensus_header< T: Serialize, U: IntoUrl, @@ -2358,7 +2359,7 @@ impl BeaconNodeHttpClient { slot: Slot, attestation_data_root: Hash256, committee_index: CommitteeIndex, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let mut path = self.eth_path(V2)?; path.path_segments_mut() diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 1acaee9654c..1ee6c437c90 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -487,6 +487,23 @@ impl<'a, E: EthSpec> From> for AttestationRef<'a, E> } } +impl ForkVersionDeserialize for Attestation { + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::Value, + fork_name: crate::ForkName, + ) -> Result { + if fork_name.electra_enabled() { + let attestation: AttestationElectra = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(Attestation::Electra(attestation)) + } else { + let attestation: AttestationBase = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(Attestation::Base(attestation)) + } + } +} + impl ForkVersionDeserialize for Vec> { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::Value, diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 5167f8b6499..30fe508a2c2 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -548,7 +548,7 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::AGGREGATES_HTTP_GET], ); - let aggregate_attestation_result = if fork_name.electra_enabled() { + if fork_name.electra_enabled() { beacon_node .get_validator_aggregate_attestation_v2( attestation_data.slot, @@ -556,6 +556,13 @@ impl AttestationService { committee_index, ) .await + .map_err(|e| { + format!("Failed to produce an aggregate attestation: {:?}", e) + })? + .ok_or_else(|| { + format!("No aggregate available for {:?}", attestation_data) + }) + .map(|result| result.data) } else { beacon_node .get_validator_aggregate_attestation_v1( @@ -563,14 +570,14 @@ impl AttestationService { attestation_data.tree_hash_root(), ) .await - }; - - aggregate_attestation_result - .map_err(|e| { - format!("Failed to produce an aggregate attestation: {:?}", e) - })? - .ok_or_else(|| format!("No aggregate available for {:?}", attestation_data)) - .map(|result| result.data) + .map_err(|e| { + format!("Failed to produce an aggregate attestation: {:?}", e) + })? + .ok_or_else(|| { + format!("No aggregate available for {:?}", attestation_data) + }) + .map(|result| result.data) + } }, ) .await From c4cb8ad8335c3e2684949ebcedf0ab7151925032 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 9 Jul 2024 08:54:58 -0700 Subject: [PATCH 205/227] fix tests --- beacon_node/http_api/tests/tests.rs | 180 +++++++++++++++++++++++----- 1 file changed, 150 insertions(+), 30 deletions(-) diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 8eabdeb36a9..ed0fdb749de 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -1664,7 +1664,7 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attestations_valid(mut self) -> Self { + pub async fn test_post_beacon_pool_attestations_valid_v1(mut self) -> Self { self.client .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await @@ -1675,6 +1675,10 @@ impl ApiTester { "valid attestation should be sent to network" ); + self + } + + pub async fn test_post_beacon_pool_attestations_valid_v2(mut self) -> Self { let fork_name = self .attestations .first() @@ -1692,7 +1696,7 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attestations_invalid(mut self) -> Self { + pub async fn test_post_beacon_pool_attestations_invalid_v1(mut self) -> Self { let mut attestations = Vec::new(); for attestation in &self.attestations { let mut invalid_attestation = attestation.clone(); @@ -1726,6 +1730,19 @@ impl ApiTester { "if some attestations are valid, we should send them to the network" ); + self + } + pub async fn test_post_beacon_pool_attestations_invalid_v2(mut self) -> Self { + let mut attestations = Vec::new(); + for attestation in &self.attestations { + let mut invalid_attestation = attestation.clone(); + invalid_attestation.data_mut().slot += 1; + + // add both to ensure we only fail on invalid attestations + attestations.push(attestation.clone()); + attestations.push(invalid_attestation); + } + let fork_name = self .attestations .first() @@ -1841,7 +1858,7 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attester_slashings_valid(mut self) -> Self { + pub async fn test_post_beacon_pool_attester_slashings_valid_v1(mut self) -> Self { self.client .post_beacon_pool_attester_slashings_v1(&self.attester_slashing) .await @@ -1852,6 +1869,10 @@ impl ApiTester { "valid attester slashing should be sent to network" ); + self + } + + pub async fn test_post_beacon_pool_attester_slashings_valid_v2(mut self) -> Self { let fork_name = self .chain .spec @@ -1869,7 +1890,7 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attester_slashings_invalid(mut self) -> Self { + pub async fn test_post_beacon_pool_attester_slashings_invalid_v1(mut self) -> Self { let mut slashing = self.attester_slashing.clone(); match &mut slashing { AttesterSlashing::Base(ref mut slashing) => { @@ -1890,6 +1911,20 @@ impl ApiTester { "invalid attester slashing should not be sent to network" ); + self + } + + pub async fn test_post_beacon_pool_attester_slashings_invalid_v2(mut self) -> Self { + let mut slashing = self.attester_slashing.clone(); + match &mut slashing { + AttesterSlashing::Base(ref mut slashing) => { + slashing.attestation_1.data.slot += 1; + } + AttesterSlashing::Electra(ref mut slashing) => { + slashing.attestation_1.data.slot += 1; + } + } + let fork_name = self .chain .spec @@ -3429,16 +3464,42 @@ impl ApiTester { ) } - pub async fn test_get_validator_aggregate_and_proofs_valid(mut self) -> Self { + pub async fn test_get_validator_aggregate_and_proofs_valid_v1(mut self) -> Self { let aggregate = self.get_aggregate().await; self.client - .post_validator_aggregate_and_proof_v1::(&[aggregate.clone()]) + .post_validator_aggregate_and_proof_v1::(&[aggregate]) .await .unwrap(); assert!(self.network_rx.network_recv.recv().await.is_some()); + self + } + + pub async fn test_get_validator_aggregate_and_proofs_invalid_v1(mut self) -> Self { + let mut aggregate = self.get_aggregate().await; + match &mut aggregate { + SignedAggregateAndProof::Base(ref mut aggregate) => { + aggregate.message.aggregate.data.slot += 1; + } + SignedAggregateAndProof::Electra(ref mut aggregate) => { + aggregate.message.aggregate.data.slot += 1; + } + } + + self.client + .post_validator_aggregate_and_proof_v1::(&[aggregate.clone()]) + .await + .unwrap_err(); + + assert!(self.network_rx.network_recv.recv().now_or_never().is_none()); + + self + } + + pub async fn test_get_validator_aggregate_and_proofs_valid_v2(mut self) -> Self { + let aggregate = self.get_aggregate().await; let fork_name = self .chain .spec @@ -3453,7 +3514,7 @@ impl ApiTester { self } - pub async fn test_get_validator_aggregate_and_proofs_invalid(mut self) -> Self { + pub async fn test_get_validator_aggregate_and_proofs_invalid_v2(mut self) -> Self { let mut aggregate = self.get_aggregate().await; match &mut aggregate { SignedAggregateAndProof::Base(ref mut aggregate) => { @@ -3464,13 +3525,6 @@ impl ApiTester { } } - self.client - .post_validator_aggregate_and_proof_v1::(&[aggregate.clone()]) - .await - .unwrap_err(); - - assert!(self.network_rx.network_recv.recv().now_or_never().is_none()); - let fork_name = self .chain .spec @@ -5888,34 +5942,66 @@ async fn post_beacon_blocks_duplicate() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attestations_valid() { +async fn beacon_pools_post_attestations_valid_v1() { + ApiTester::new() + .await + .test_post_beacon_pool_attestations_valid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attestations_invalid_v1() { + ApiTester::new() + .await + .test_post_beacon_pool_attestations_invalid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attestations_valid_v2() { + ApiTester::new() + .await + .test_post_beacon_pool_attestations_valid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attestations_invalid_v2() { + ApiTester::new() + .await + .test_post_beacon_pool_attestations_invalid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attester_slashings_valid_v1() { ApiTester::new() .await - .test_post_beacon_pool_attestations_valid() + .test_post_beacon_pool_attester_slashings_valid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attestations_invalid() { +async fn beacon_pools_post_attester_slashings_invalid_v1() { ApiTester::new() .await - .test_post_beacon_pool_attestations_invalid() + .test_post_beacon_pool_attester_slashings_invalid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attester_slashings_valid() { +async fn beacon_pools_post_attester_slashings_valid_v2() { ApiTester::new() .await - .test_post_beacon_pool_attester_slashings_valid() + .test_post_beacon_pool_attester_slashings_valid_v2() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attester_slashings_invalid() { +async fn beacon_pools_post_attester_slashings_invalid_v2() { ApiTester::new() .await - .test_post_beacon_pool_attester_slashings_invalid() + .test_post_beacon_pool_attester_slashings_invalid_v2() .await; } @@ -6243,36 +6329,70 @@ async fn get_validator_aggregate_attestation_with_skip_slots() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_valid() { +async fn get_validator_aggregate_and_proofs_valid_v1() { + ApiTester::new() + .await + .test_get_validator_aggregate_and_proofs_valid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_valid_with_skip_slots_v1() { + ApiTester::new() + .await + .skip_slots(E::slots_per_epoch() * 2) + .test_get_validator_aggregate_and_proofs_valid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_valid_v2() { + ApiTester::new() + .await + .test_get_validator_aggregate_and_proofs_valid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_valid_with_skip_slots_v2() { + ApiTester::new() + .await + .skip_slots(E::slots_per_epoch() * 2) + .test_get_validator_aggregate_and_proofs_valid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_invalid_v1() { ApiTester::new() .await - .test_get_validator_aggregate_and_proofs_valid() + .test_get_validator_aggregate_and_proofs_invalid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_valid_with_skip_slots() { +async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots_v1() { ApiTester::new() .await .skip_slots(E::slots_per_epoch() * 2) - .test_get_validator_aggregate_and_proofs_valid() + .test_get_validator_aggregate_and_proofs_invalid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_invalid() { +async fn get_validator_aggregate_and_proofs_invalid_v2() { ApiTester::new() .await - .test_get_validator_aggregate_and_proofs_invalid() + .test_get_validator_aggregate_and_proofs_invalid_v2() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots() { +async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots_v2() { ApiTester::new() .await .skip_slots(E::slots_per_epoch() * 2) - .test_get_validator_aggregate_and_proofs_invalid() + .test_get_validator_aggregate_and_proofs_invalid_v2() .await; } From 386aacda2ab6032ed84fbea7b7d847b0cb5a86d6 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 11 Jul 2024 11:38:10 -0700 Subject: [PATCH 206/227] convert op pool messages to electra in electra --- beacon_node/http_api/src/lib.rs | 73 +++++++++++++++++------- consensus/types/src/attestation.rs | 35 ++++++++++++ consensus/types/src/attester_slashing.rs | 15 ++++- 3 files changed, 102 insertions(+), 21 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 6656485dceb..3adf84fee13 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1834,18 +1834,35 @@ pub fn serve( .filter(|&att| query_filter(att.data())) .cloned(), ); - let slot = query - .slot - .or_else(|| { - attestations - .first() - .map(|att| att.data().slot) - .or_else(|| chain.slot_clock.now()) - }) - .ok_or(warp_utils::reject::custom_server_error( - "unable to read slot clock".to_string(), - ))?; - let fork_name = chain.spec.fork_name_at_slot::(slot); + // Use the current slot to find the fork version, and convert all messages to the + // current fork's format. This is to ensure consistent message types matching + // `Eth-Consensus-Version`. + let current_slot = + chain + .slot_clock + .now() + .ok_or(warp_utils::reject::custom_server_error( + "unable to read slot clock".to_string(), + ))?; + let fork_name = chain.spec.fork_name_at_slot::(current_slot); + + let attestations = if fork_name.electra_enabled() { + attestations + .into_iter() + .map(|att| match att { + Attestation::Base(a) => Ok(Attestation::Electra(a.try_into()?)), + Attestation::Electra(a) => Ok(Attestation::Electra(a)), + }) + .collect::, types::attestation::Error>>() + .map_err(|e| { + warp_utils::reject::custom_server_error(format!( + "could not convert base attestations to electra {e:?}" + )) + })? + } else { + attestations + }; + let res = fork_versioned_response(endpoint_version, fork_name, &attestations)?; Ok(add_consensus_version_header( warp::reply::json(&res).into_response(), @@ -1914,14 +1931,30 @@ pub fn serve( chain: Arc>| { task_spawner.blocking_response_task(Priority::P1, move || { let slashings = chain.op_pool.get_all_attester_slashings(); - let slot = slashings - .first() - .map(|slashing| slashing.attestation_1().data().slot) - .or_else(|| chain.slot_clock.now()) - .ok_or(warp_utils::reject::custom_server_error( - "unable to read slot clock".to_string(), - ))?; - let fork_name = chain.spec.fork_name_at_slot::(slot); + + // Use the current slot to find the fork version, and convert all messages to the + // current fork's format. This is to ensure consistent message types matching + // `Eth-Consensus-Version`. + let current_slot = + chain + .slot_clock + .now() + .ok_or(warp_utils::reject::custom_server_error( + "unable to read slot clock".to_string(), + ))?; + let fork_name = chain.spec.fork_name_at_slot::(current_slot); + + let slashings = if fork_name.electra_enabled() { + slashings + .into_iter() + .map(|att| match att { + AttesterSlashing::Base(a) => AttesterSlashing::Electra(a.into()), + AttesterSlashing::Electra(a) => AttesterSlashing::Electra(a), + }) + .collect::>() + } else { + slashings + }; let res = fork_versioned_response(endpoint_version, fork_name, &slashings)?; Ok(add_consensus_version_header( warp::reply::json(&res).into_response(), diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 1ee6c437c90..f12f84a447d 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -26,6 +26,12 @@ pub enum Error { InvalidCommitteeIndex, } +impl From for Error { + fn from(e: ssz_types::Error) -> Self { + Error::SszTypesError(e) + } +} + #[superstruct( variants(Base, Electra), variant_attributes( @@ -416,6 +422,35 @@ impl AttestationBase { } } +impl TryFrom> for AttestationElectra { + type Error = Error; + fn try_from(att: AttestationBase) -> Result { + // Extend the aggregation bits list. + let aggregation_bits = att.extend_aggregation_bits()?; + let AttestationBase { + aggregation_bits: _, + mut data, + signature, + } = att; + + // Set the committee index based on the index field. + let mut committee_bits: BitVector = BitVector::default(); + committee_bits + .set(data.index as usize, true) + .map_err(|_| Error::InvalidCommitteeIndex)?; + + // Set the attestation data's index to zero. + data.index = 0; + + Ok(Self { + aggregation_bits, + data, + committee_bits, + signature, + }) + } +} + impl SlotData for Attestation { fn get_slot(&self) -> Slot { self.data().slot diff --git a/consensus/types/src/attester_slashing.rs b/consensus/types/src/attester_slashing.rs index f6aa654d445..18ab63ad44e 100644 --- a/consensus/types/src/attester_slashing.rs +++ b/consensus/types/src/attester_slashing.rs @@ -1,5 +1,5 @@ use crate::indexed_attestation::{ - IndexedAttestationBase, IndexedAttestationElectra, IndexedAttestationRef, + IndexedAttestation, IndexedAttestationBase, IndexedAttestationElectra, IndexedAttestationRef, }; use crate::{test_utils::TestRandom, EthSpec}; use derivative::Derivative; @@ -161,6 +161,19 @@ impl AttesterSlashing { } } +impl From> for AttesterSlashingElectra { + fn from(attester_slashing: AttesterSlashingBase) -> Self { + let AttesterSlashingBase { + attestation_1, + attestation_2, + } = attester_slashing; + AttesterSlashingElectra { + attestation_1: IndexedAttestation::Base(attestation_1).to_electra(), + attestation_2: IndexedAttestation::Base(attestation_2).to_electra(), + } + } +} + impl TestRandom for AttesterSlashing { fn random_for_test(rng: &mut impl RngCore) -> Self { if rng.gen_bool(0.5) { From 71a2eadc467040a4a2996177e557673e5dc104da Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 11 Jul 2024 12:17:29 -0700 Subject: [PATCH 207/227] remove methods to post without content header --- common/eth2/src/lib.rs | 56 +++--------------------------------------- 1 file changed, 3 insertions(+), 53 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index e6fa6eb02f3..af99cddda6b 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -389,53 +389,6 @@ impl BeaconNodeHttpClient { ok_or_error(response).await } - /// Generic POST function supporting arbitrary responses and timeouts. - /// Does not include Content-Type application/json in the request header. - async fn post_generic_json_without_content_type_header( - &self, - url: U, - body: &T, - timeout: Option, - ) -> Result { - let mut builder = self.client.post(url); - if let Some(timeout) = timeout { - builder = builder.timeout(timeout); - } - - let serialized_body = serde_json::to_vec(body).map_err(Error::InvalidJson)?; - - let response = builder.body(serialized_body).send().await?; - ok_or_error(response).await - } - - /// Generic POST function supporting arbitrary responses and timeouts. - /// Does not include Content-Type application/json in the request header. - /// Does include Eth-Consensus-Version in the request header. - async fn post_generic_json_without_content_type_header_but_with_consensus_header< - T: Serialize, - U: IntoUrl, - >( - &self, - url: U, - body: &T, - timeout: Option, - fork: ForkName, - ) -> Result { - let mut builder = self.client.post(url); - if let Some(timeout) = timeout { - builder = builder.timeout(timeout); - } - - let serialized_body = serde_json::to_vec(body).map_err(Error::InvalidJson)?; - - let response = builder - .header(CONSENSUS_VERSION_HEADER, fork.to_string()) - .body(serialized_body) - .send() - .await?; - ok_or_error(response).await - } - /// Generic POST function supporting arbitrary responses and timeouts. async fn post_generic_with_consensus_version( &self, @@ -1382,8 +1335,7 @@ impl BeaconNodeHttpClient { .push("pool") .push("attester_slashings"); - self.post_generic_json_without_content_type_header(path, slashing, None) - .await?; + self.post_generic(path, slashing, None).await?; Ok(()) } @@ -1402,10 +1354,8 @@ impl BeaconNodeHttpClient { .push("pool") .push("attester_slashings"); - self.post_generic_json_without_content_type_header_but_with_consensus_header( - path, slashing, None, fork_name, - ) - .await?; + self.post_generic_with_consensus_version(path, slashing, None, fork_name) + .await?; Ok(()) } From e3ce7fc5eae558da117b41ca7abbd6e03f4e420d Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 15 Jul 2024 17:36:05 +0200 Subject: [PATCH 208/227] Add BeaconBlocksByRange v3 --- beacon_node/network/src/sync/block_lookups/mod.rs | 2 +- beacon_node/network/src/sync/range_sync/chain.rs | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 0a44cf2fdf5..43bae1f9601 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -53,7 +53,7 @@ mod tests; /// The maximum depth we will search for a parent block. In principle we should have sync'd any /// canonical chain to its head once the peer connects. A chain should not appear where it's depth /// is further back than the most recent head slot. -pub(crate) const PARENT_DEPTH_TOLERANCE: usize = SLOT_IMPORT_TOLERANCE * 2; +pub(crate) const PARENT_DEPTH_TOLERANCE: usize = SLOT_IMPORT_TOLERANCE + 1; const FAILED_CHAINS_CACHE_EXPIRY_SECONDS: u64 = 60; pub const SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS: u8 = 4; diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index a735001fed3..0e4058dddf1 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -11,6 +11,7 @@ use rand::{seq::SliceRandom, Rng}; use slog::{crit, debug, o, warn}; use std::collections::{btree_map::Entry, BTreeMap, HashSet}; use std::hash::{Hash, Hasher}; +use std::time::Instant; use types::{Epoch, EthSpec, Hash256, Slot}; /// Blocks are downloaded in batches from peers. This constant specifies how many epochs worth of @@ -109,6 +110,9 @@ pub struct SyncingChain { pub enum ChainSyncingState { /// The chain is not being synced. Stopped, + /// The chain should not download any more batches, but should attempt to processing everything + /// that is already downloaded. + Stopping(Instant), /// The chain is undergoing syncing. Syncing, } @@ -866,6 +870,11 @@ impl SyncingChain { network: &mut SyncNetworkContext, batch_id: BatchId, ) -> ProcessingResult { + // If chain is stopping do not request any more batches. + if matches!(self.state, ChainSyncingState::Stopping) { + return Ok(KeepChain); + } + let Some(batch) = self.batches.get_mut(&batch_id) else { return Ok(KeepChain); }; @@ -972,7 +981,7 @@ impl SyncingChain { pub fn is_syncing(&self) -> bool { match self.state { ChainSyncingState::Syncing => true, - ChainSyncingState::Stopped => false, + ChainSyncingState::Stopped | ChainSyncingState::Stopping { .. } => false, } } @@ -991,6 +1000,7 @@ impl SyncingChain { /// Attempts to request the next required batches from the peer pool if the chain is syncing. It will exhaust the peer /// pool and left over batches until the batch buffer is reached or all peers are exhausted. fn request_batches(&mut self, network: &mut SyncNetworkContext) -> ProcessingResult { + // If chain is stopped or stopping do not request any more batches. if !matches!(self.state, ChainSyncingState::Syncing) { return Ok(KeepChain); } From 9f40d91d513ac96ba4ab7758083b0fef57b640aa Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 15 Jul 2024 18:01:06 +0200 Subject: [PATCH 209/227] Revert "Add BeaconBlocksByRange v3" This reverts commit e3ce7fc5eae558da117b41ca7abbd6e03f4e420d. --- beacon_node/network/src/sync/block_lookups/mod.rs | 2 +- beacon_node/network/src/sync/range_sync/chain.rs | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 43bae1f9601..0a44cf2fdf5 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -53,7 +53,7 @@ mod tests; /// The maximum depth we will search for a parent block. In principle we should have sync'd any /// canonical chain to its head once the peer connects. A chain should not appear where it's depth /// is further back than the most recent head slot. -pub(crate) const PARENT_DEPTH_TOLERANCE: usize = SLOT_IMPORT_TOLERANCE + 1; +pub(crate) const PARENT_DEPTH_TOLERANCE: usize = SLOT_IMPORT_TOLERANCE * 2; const FAILED_CHAINS_CACHE_EXPIRY_SECONDS: u64 = 60; pub const SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS: u8 = 4; diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index 0e4058dddf1..a735001fed3 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -11,7 +11,6 @@ use rand::{seq::SliceRandom, Rng}; use slog::{crit, debug, o, warn}; use std::collections::{btree_map::Entry, BTreeMap, HashSet}; use std::hash::{Hash, Hasher}; -use std::time::Instant; use types::{Epoch, EthSpec, Hash256, Slot}; /// Blocks are downloaded in batches from peers. This constant specifies how many epochs worth of @@ -110,9 +109,6 @@ pub struct SyncingChain { pub enum ChainSyncingState { /// The chain is not being synced. Stopped, - /// The chain should not download any more batches, but should attempt to processing everything - /// that is already downloaded. - Stopping(Instant), /// The chain is undergoing syncing. Syncing, } @@ -870,11 +866,6 @@ impl SyncingChain { network: &mut SyncNetworkContext, batch_id: BatchId, ) -> ProcessingResult { - // If chain is stopping do not request any more batches. - if matches!(self.state, ChainSyncingState::Stopping) { - return Ok(KeepChain); - } - let Some(batch) = self.batches.get_mut(&batch_id) else { return Ok(KeepChain); }; @@ -981,7 +972,7 @@ impl SyncingChain { pub fn is_syncing(&self) -> bool { match self.state { ChainSyncingState::Syncing => true, - ChainSyncingState::Stopped | ChainSyncingState::Stopping { .. } => false, + ChainSyncingState::Stopped => false, } } @@ -1000,7 +991,6 @@ impl SyncingChain { /// Attempts to request the next required batches from the peer pool if the chain is syncing. It will exhaust the peer /// pool and left over batches until the batch buffer is reached or all peers are exhausted. fn request_batches(&mut self, network: &mut SyncNetworkContext) -> ProcessingResult { - // If chain is stopped or stopping do not request any more batches. if !matches!(self.state, ChainSyncingState::Syncing) { return Ok(KeepChain); } From 4065ef66abad8ef76032844905f7e4a9c4310b39 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 15 Jul 2024 10:57:13 -0700 Subject: [PATCH 210/227] filter instead of convert --- beacon_node/http_api/src/lib.rs | 100 ++++++++++------------- consensus/types/src/attestation.rs | 29 ------- consensus/types/src/attester_slashing.rs | 15 +--- 3 files changed, 45 insertions(+), 99 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 3adf84fee13..2d017d65391 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1845,23 +1845,14 @@ pub fn serve( "unable to read slot clock".to_string(), ))?; let fork_name = chain.spec.fork_name_at_slot::(current_slot); - - let attestations = if fork_name.electra_enabled() { - attestations - .into_iter() - .map(|att| match att { - Attestation::Base(a) => Ok(Attestation::Electra(a.try_into()?)), - Attestation::Electra(a) => Ok(Attestation::Electra(a)), - }) - .collect::, types::attestation::Error>>() - .map_err(|e| { - warp_utils::reject::custom_server_error(format!( - "could not convert base attestations to electra {e:?}" - )) - })? - } else { - attestations - }; + let attestations = attestations + .into_iter() + .filter(|att| { + (fork_name.electra_enabled() && matches!(att, Attestation::Electra(_))) + || (!fork_name.electra_enabled() + && matches!(att, Attestation::Base(_))) + }) + .collect::>(); let res = fork_versioned_response(endpoint_version, fork_name, &attestations)?; Ok(add_consensus_version_header( @@ -1921,48 +1912,45 @@ pub fn serve( ); // GET beacon/pool/attester_slashings - let get_beacon_pool_attester_slashings = beacon_pool_path_any - .clone() - .and(warp::path("attester_slashings")) - .and(warp::path::end()) - .then( - |endpoint_version: EndpointVersion, - task_spawner: TaskSpawner, - chain: Arc>| { - task_spawner.blocking_response_task(Priority::P1, move || { - let slashings = chain.op_pool.get_all_attester_slashings(); - - // Use the current slot to find the fork version, and convert all messages to the - // current fork's format. This is to ensure consistent message types matching - // `Eth-Consensus-Version`. - let current_slot = - chain - .slot_clock - .now() - .ok_or(warp_utils::reject::custom_server_error( + let get_beacon_pool_attester_slashings = + beacon_pool_path_any + .clone() + .and(warp::path("attester_slashings")) + .and(warp::path::end()) + .then( + |endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_response_task(Priority::P1, move || { + let slashings = chain.op_pool.get_all_attester_slashings(); + + // Use the current slot to find the fork version, and convert all messages to the + // current fork's format. This is to ensure consistent message types matching + // `Eth-Consensus-Version`. + let current_slot = chain.slot_clock.now().ok_or( + warp_utils::reject::custom_server_error( "unable to read slot clock".to_string(), - ))?; - let fork_name = chain.spec.fork_name_at_slot::(current_slot); - - let slashings = if fork_name.electra_enabled() { - slashings + ), + )?; + let fork_name = chain.spec.fork_name_at_slot::(current_slot); + let slashings = slashings .into_iter() - .map(|att| match att { - AttesterSlashing::Base(a) => AttesterSlashing::Electra(a.into()), - AttesterSlashing::Electra(a) => AttesterSlashing::Electra(a), + .filter(|slashing| { + (fork_name.electra_enabled() + && matches!(slashing, AttesterSlashing::Electra(_))) + || (!fork_name.electra_enabled() + && matches!(slashing, AttesterSlashing::Base(_))) }) - .collect::>() - } else { - slashings - }; - let res = fork_versioned_response(endpoint_version, fork_name, &slashings)?; - Ok(add_consensus_version_header( - warp::reply::json(&res).into_response(), - fork_name, - )) - }) - }, - ); + .collect::>(); + + let res = fork_versioned_response(endpoint_version, fork_name, &slashings)?; + Ok(add_consensus_version_header( + warp::reply::json(&res).into_response(), + fork_name, + )) + }) + }, + ); // POST beacon/pool/proposer_slashings let post_beacon_pool_proposer_slashings = beacon_pool_path diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index f12f84a447d..7b53a98caa1 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -422,35 +422,6 @@ impl AttestationBase { } } -impl TryFrom> for AttestationElectra { - type Error = Error; - fn try_from(att: AttestationBase) -> Result { - // Extend the aggregation bits list. - let aggregation_bits = att.extend_aggregation_bits()?; - let AttestationBase { - aggregation_bits: _, - mut data, - signature, - } = att; - - // Set the committee index based on the index field. - let mut committee_bits: BitVector = BitVector::default(); - committee_bits - .set(data.index as usize, true) - .map_err(|_| Error::InvalidCommitteeIndex)?; - - // Set the attestation data's index to zero. - data.index = 0; - - Ok(Self { - aggregation_bits, - data, - committee_bits, - signature, - }) - } -} - impl SlotData for Attestation { fn get_slot(&self) -> Slot { self.data().slot diff --git a/consensus/types/src/attester_slashing.rs b/consensus/types/src/attester_slashing.rs index 18ab63ad44e..f6aa654d445 100644 --- a/consensus/types/src/attester_slashing.rs +++ b/consensus/types/src/attester_slashing.rs @@ -1,5 +1,5 @@ use crate::indexed_attestation::{ - IndexedAttestation, IndexedAttestationBase, IndexedAttestationElectra, IndexedAttestationRef, + IndexedAttestationBase, IndexedAttestationElectra, IndexedAttestationRef, }; use crate::{test_utils::TestRandom, EthSpec}; use derivative::Derivative; @@ -161,19 +161,6 @@ impl AttesterSlashing { } } -impl From> for AttesterSlashingElectra { - fn from(attester_slashing: AttesterSlashingBase) -> Self { - let AttesterSlashingBase { - attestation_1, - attestation_2, - } = attester_slashing; - AttesterSlashingElectra { - attestation_1: IndexedAttestation::Base(attestation_1).to_electra(), - attestation_2: IndexedAttestation::Base(attestation_2).to_electra(), - } - } -} - impl TestRandom for AttesterSlashing { fn random_for_test(rng: &mut impl RngCore) -> Self { if rng.gen_bool(0.5) { From 0e5993943ef12e99248163c5a8c08f37cfc5df26 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 15 Jul 2024 20:51:59 +0200 Subject: [PATCH 211/227] Add range sync metrics to track efficiency (#6095) * Add more range sync metrics to track efficiency * Add ignored blocks metrics --- beacon_node/network/src/metrics.rs | 30 +++++++++ .../network_beacon_processor/sync_methods.rs | 14 +++-- .../network/src/sync/backfill_sync/mod.rs | 10 +-- beacon_node/network/src/sync/manager.rs | 5 +- .../network/src/sync/range_sync/batch.rs | 28 ++++++--- .../network/src/sync/range_sync/chain.rs | 62 +++++++++++++++++-- .../src/sync/range_sync/chain_collection.rs | 23 +++++-- .../network/src/sync/range_sync/range.rs | 7 +++ 8 files changed, 151 insertions(+), 28 deletions(-) diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index dbcc8fb9b4c..f0dba8d9655 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -237,6 +237,36 @@ lazy_static! { "Number of Syncing chains in range, per range type", &["range_type"] ); + pub static ref SYNCING_CHAINS_REMOVED: Result = try_create_int_counter_vec( + "sync_range_removed_chains_total", + "Total count of range syncing chains removed per range type", + &["range_type"] + ); + pub static ref SYNCING_CHAINS_ADDED: Result = try_create_int_counter_vec( + "sync_range_added_chains_total", + "Total count of range syncing chains added per range type", + &["range_type"] + ); + pub static ref SYNCING_CHAINS_DROPPED_BLOCKS: Result = try_create_int_counter_vec( + "sync_range_chains_dropped_blocks_total", + "Total count of dropped blocks when removing a syncing chain per range type", + &["range_type"] + ); + pub static ref SYNCING_CHAINS_IGNORED_BLOCKS: Result = try_create_int_counter_vec( + "sync_range_chains_ignored_blocks_total", + "Total count of ignored blocks when processing a syncing chain batch per chain type", + &["chain_type"] + ); + pub static ref SYNCING_CHAINS_PROCESSED_BATCHES: Result = try_create_int_counter_vec( + "sync_range_chains_processed_batches_total", + "Total count of processed batches in a syncing chain batch per chain type", + &["chain_type"] + ); + pub static ref SYNCING_CHAIN_BATCH_AWAITING_PROCESSING: Result = try_create_histogram_with_buckets( + "sync_range_chain_batch_awaiting_processing_seconds", + "Time range sync batches spend in AwaitingProcessing state", + Ok(vec![0.01,0.02,0.05,0.1,0.2,0.5,1.0,2.0,5.0,10.0,20.0]) + ); pub static ref SYNC_SINGLE_BLOCK_LOOKUPS: Result = try_create_int_gauge( "sync_single_block_lookups", "Number of single block lookups underway" diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index acd02ab6ad8..68bd6745144 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -326,7 +326,7 @@ impl NetworkBeaconProcessor { .process_blocks(downloaded_blocks.iter(), notify_execution_layer) .await { - (_, Ok(_)) => { + (imported_blocks, Ok(_)) => { debug!(self.log, "Batch processed"; "batch_epoch" => epoch, "first_block_slot" => start_slot, @@ -335,7 +335,8 @@ impl NetworkBeaconProcessor { "processed_blocks" => sent_blocks, "service"=> "sync"); BatchProcessResult::Success { - was_non_empty: sent_blocks > 0, + sent_blocks, + imported_blocks, } } (imported_blocks, Err(e)) => { @@ -349,7 +350,7 @@ impl NetworkBeaconProcessor { "service" => "sync"); match e.peer_action { Some(penalty) => BatchProcessResult::FaultyFailure { - imported_blocks: imported_blocks > 0, + imported_blocks, penalty, }, None => BatchProcessResult::NonFaultyFailure, @@ -368,7 +369,7 @@ impl NetworkBeaconProcessor { .sum::(); match self.process_backfill_blocks(downloaded_blocks) { - (_, Ok(_)) => { + (imported_blocks, Ok(_)) => { debug!(self.log, "Backfill batch processed"; "batch_epoch" => epoch, "first_block_slot" => start_slot, @@ -377,7 +378,8 @@ impl NetworkBeaconProcessor { "processed_blobs" => n_blobs, "service"=> "sync"); BatchProcessResult::Success { - was_non_empty: sent_blocks > 0, + sent_blocks, + imported_blocks, } } (_, Err(e)) => { @@ -390,7 +392,7 @@ impl NetworkBeaconProcessor { "service" => "sync"); match e.peer_action { Some(penalty) => BatchProcessResult::FaultyFailure { - imported_blocks: false, + imported_blocks: 0, penalty, }, None => BatchProcessResult::NonFaultyFailure, diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 8bcc95fd3ce..dfb05da19bd 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -528,7 +528,7 @@ impl BackFillSync { // result callback. This is done, because an empty batch could end a chain and the logic // for removing chains and checking completion is in the callback. - let blocks = match batch.start_processing() { + let (blocks, _) = match batch.start_processing() { Err(e) => { return self .fail_sync(BackFillError::BatchInvalidState(batch_id, e.0)) @@ -615,13 +615,15 @@ impl BackFillSync { "batch_epoch" => batch_id, "peer" => %peer, "client" => %network.client_type(&peer)); match result { - BatchProcessResult::Success { was_non_empty } => { + BatchProcessResult::Success { + imported_blocks, .. + } => { if let Err(e) = batch.processing_completed(BatchProcessingResult::Success) { self.fail_sync(BackFillError::BatchInvalidState(batch_id, e.0))?; } // If the processed batch was not empty, we can validate previous unvalidated // blocks. - if *was_non_empty { + if *imported_blocks > 0 { self.advance_chain(network, batch_id); } @@ -677,7 +679,7 @@ impl BackFillSync { Ok(BatchOperationOutcome::Continue) => { // chain can continue. Check if it can be progressed - if *imported_blocks { + if *imported_blocks > 0 { // At least one block was successfully verified and imported, then we can be sure all // previous batches are valid and we only need to download the current failed // batch. diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 23c05a6e161..dd4fa56d537 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -156,11 +156,12 @@ pub enum BlockProcessingResult { pub enum BatchProcessResult { /// The batch was completed successfully. It carries whether the sent batch contained blocks. Success { - was_non_empty: bool, + sent_blocks: usize, + imported_blocks: usize, }, /// The batch processing failed. It carries whether the processing imported any block. FaultyFailure { - imported_blocks: bool, + imported_blocks: usize, penalty: PeerAction, }, NonFaultyFailure, diff --git a/beacon_node/network/src/sync/range_sync/batch.rs b/beacon_node/network/src/sync/range_sync/batch.rs index 6e377cc6cb4..49e3ac3a817 100644 --- a/beacon_node/network/src/sync/range_sync/batch.rs +++ b/beacon_node/network/src/sync/range_sync/batch.rs @@ -5,6 +5,7 @@ use lighthouse_network::PeerId; use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::ops::Sub; +use std::time::{Duration, Instant}; use strum::Display; use types::{Epoch, EthSpec, Slot}; @@ -118,7 +119,7 @@ pub enum BatchState { /// The batch is being downloaded. Downloading(PeerId, Id), /// The batch has been completely downloaded and is ready for processing. - AwaitingProcessing(PeerId, Vec>), + AwaitingProcessing(PeerId, Vec>, Instant), /// The batch is being processed. Processing(Attempt), /// The batch was successfully processed and is waiting to be validated. @@ -210,13 +211,26 @@ impl BatchInfo { match &self.state { BatchState::AwaitingDownload | BatchState::Failed => None, BatchState::Downloading(peer_id, _) - | BatchState::AwaitingProcessing(peer_id, _) + | BatchState::AwaitingProcessing(peer_id, _, _) | BatchState::Processing(Attempt { peer_id, .. }) | BatchState::AwaitingValidation(Attempt { peer_id, .. }) => Some(peer_id), BatchState::Poisoned => unreachable!("Poisoned batch"), } } + /// Returns the count of stored pending blocks if in awaiting processing state + pub fn pending_blocks(&self) -> usize { + match &self.state { + BatchState::AwaitingProcessing(_, blocks, _) => blocks.len(), + BatchState::AwaitingDownload + | BatchState::Downloading { .. } + | BatchState::Processing { .. } + | BatchState::AwaitingValidation { .. } + | BatchState::Poisoned + | BatchState::Failed => 0, + } + } + /// Returns a BlocksByRange request associated with the batch. pub fn to_blocks_by_range_request(&self) -> (BlocksByRangeRequest, ByRangeRequestType) { ( @@ -293,7 +307,7 @@ impl BatchInfo { } let received = blocks.len(); - self.state = BatchState::AwaitingProcessing(peer, blocks); + self.state = BatchState::AwaitingProcessing(peer, blocks, Instant::now()); Ok(received) } BatchState::Poisoned => unreachable!("Poisoned batch"), @@ -365,11 +379,11 @@ impl BatchInfo { } } - pub fn start_processing(&mut self) -> Result>, WrongState> { + pub fn start_processing(&mut self) -> Result<(Vec>, Duration), WrongState> { match self.state.poison() { - BatchState::AwaitingProcessing(peer, blocks) => { + BatchState::AwaitingProcessing(peer, blocks, start_instant) => { self.state = BatchState::Processing(Attempt::new::(peer, &blocks)); - Ok(blocks) + Ok((blocks, start_instant.elapsed())) } BatchState::Poisoned => unreachable!("Poisoned batch"), other => { @@ -515,7 +529,7 @@ impl std::fmt::Debug for BatchState { }) => write!(f, "AwaitingValidation({})", peer_id), BatchState::AwaitingDownload => f.write_str("AwaitingDownload"), BatchState::Failed => f.write_str("Failed"), - BatchState::AwaitingProcessing(ref peer, ref blocks) => { + BatchState::AwaitingProcessing(ref peer, ref blocks, _) => { write!(f, "AwaitingProcessing({}, {} blocks)", peer, blocks.len()) } BatchState::Downloading(peer, request_id) => { diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index a735001fed3..d63b2f95d80 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -1,4 +1,6 @@ use super::batch::{BatchInfo, BatchProcessingResult, BatchState}; +use super::RangeSyncType; +use crate::metrics; use crate::network_beacon_processor::ChainSegmentProcessId; use crate::sync::network_context::RangeRequestId; use crate::sync::{network_context::SyncNetworkContext, BatchOperationOutcome, BatchProcessResult}; @@ -11,6 +13,7 @@ use rand::{seq::SliceRandom, Rng}; use slog::{crit, debug, o, warn}; use std::collections::{btree_map::Entry, BTreeMap, HashSet}; use std::hash::{Hash, Hasher}; +use strum::IntoStaticStr; use types::{Epoch, EthSpec, Hash256, Slot}; /// Blocks are downloaded in batches from peers. This constant specifies how many epochs worth of @@ -53,6 +56,13 @@ pub struct KeepChain; pub type ChainId = u64; pub type BatchId = Epoch; +#[derive(Debug, Copy, Clone, IntoStaticStr)] +pub enum SyncingChainType { + Head, + Finalized, + Backfill, +} + /// A chain of blocks that need to be downloaded. Peers who claim to contain the target head /// root are grouped into the peer pool and queried for batches when downloading the /// chain. @@ -60,6 +70,9 @@ pub struct SyncingChain { /// A random id used to identify this chain. id: ChainId, + /// SyncingChain type + pub chain_type: SyncingChainType, + /// The start of the chain segment. Any epoch previous to this one has been validated. pub start_epoch: Epoch, @@ -126,6 +139,7 @@ impl SyncingChain { target_head_slot: Slot, target_head_root: Hash256, peer_id: PeerId, + chain_type: SyncingChainType, log: &slog::Logger, ) -> Self { let mut peers = FnvHashMap::default(); @@ -135,6 +149,7 @@ impl SyncingChain { SyncingChain { id, + chain_type, start_epoch, target_head_slot, target_head_root, @@ -171,6 +186,14 @@ impl SyncingChain { self.validated_batches * EPOCHS_PER_BATCH } + /// Returns the total count of pending blocks in all the batches of this chain + pub fn pending_blocks(&self) -> usize { + self.batches + .values() + .map(|batch| batch.pending_blocks()) + .sum() + } + /// Removes a peer from the chain. /// If the peer has active batches, those are considered failed and re-requested. pub fn remove_peer( @@ -305,7 +328,12 @@ impl SyncingChain { // result callback. This is done, because an empty batch could end a chain and the logic // for removing chains and checking completion is in the callback. - let blocks = batch.start_processing()?; + let (blocks, duration_in_awaiting_processing) = batch.start_processing()?; + metrics::observe_duration( + &metrics::SYNCING_CHAIN_BATCH_AWAITING_PROCESSING, + duration_in_awaiting_processing, + ); + let process_id = ChainSegmentProcessId::RangeBatchId(self.id, batch_id); self.current_processing_batch = Some(batch_id); @@ -469,10 +497,27 @@ impl SyncingChain { // We consider three cases. Batch was successfully processed, Batch failed processing due // to a faulty peer, or batch failed processing but the peer can't be deemed faulty. match result { - BatchProcessResult::Success { was_non_empty } => { + BatchProcessResult::Success { + sent_blocks, + imported_blocks, + } => { + if sent_blocks > imported_blocks { + let ignored_blocks = sent_blocks - imported_blocks; + metrics::inc_counter_vec_by( + &metrics::SYNCING_CHAINS_IGNORED_BLOCKS, + &[self.chain_type.into()], + ignored_blocks as u64, + ); + } + metrics::inc_counter_vec( + &metrics::SYNCING_CHAINS_PROCESSED_BATCHES, + &[self.chain_type.into()], + ); + batch.processing_completed(BatchProcessingResult::Success)?; - if *was_non_empty { + // was not empty = sent_blocks > 0 + if *sent_blocks > 0 { // If the processed batch was not empty, we can validate previous unvalidated // blocks. self.advance_chain(network, batch_id); @@ -515,7 +560,7 @@ impl SyncingChain { match batch.processing_completed(BatchProcessingResult::FaultyFailure)? { BatchOperationOutcome::Continue => { // Chain can continue. Check if it can be moved forward. - if *imported_blocks { + if *imported_blocks > 0 { // At least one block was successfully verified and imported, so we can be sure all // previous batches are valid and we only need to download the current failed // batch. @@ -1142,3 +1187,12 @@ impl RemoveChain { ) } } + +impl From for SyncingChainType { + fn from(value: RangeSyncType) -> Self { + match value { + RangeSyncType::Head => Self::Head, + RangeSyncType::Finalized => Self::Finalized, + } + } +} diff --git a/beacon_node/network/src/sync/range_sync/chain_collection.rs b/beacon_node/network/src/sync/range_sync/chain_collection.rs index 364514a3582..3621a6605af 100644 --- a/beacon_node/network/src/sync/range_sync/chain_collection.rs +++ b/beacon_node/network/src/sync/range_sync/chain_collection.rs @@ -64,8 +64,8 @@ impl ChainCollection { /// Updates the Syncing state of the collection after a chain is removed. fn on_chain_removed(&mut self, id: &ChainId, was_syncing: bool, sync_type: RangeSyncType) { - let _ = metrics::get_int_gauge(&metrics::SYNCING_CHAINS_COUNT, &[sync_type.as_str()]) - .map(|m| m.dec()); + metrics::inc_counter_vec(&metrics::SYNCING_CHAINS_REMOVED, &[sync_type.as_str()]); + self.update_metrics(); match self.state { RangeSyncState::Finalized(ref syncing_id) => { @@ -493,15 +493,28 @@ impl ChainCollection { target_head_slot, target_head_root, peer, + sync_type.into(), &self.log, ); debug_assert_eq!(new_chain.get_id(), id); debug!(self.log, "New chain added to sync"; "peer_id" => peer_rpr, "sync_type" => ?sync_type, &new_chain); entry.insert(new_chain); - let _ = - metrics::get_int_gauge(&metrics::SYNCING_CHAINS_COUNT, &[sync_type.as_str()]) - .map(|m| m.inc()); + metrics::inc_counter_vec(&metrics::SYNCING_CHAINS_ADDED, &[sync_type.as_str()]); + self.update_metrics(); } } } + + fn update_metrics(&self) { + metrics::set_gauge_vec( + &metrics::SYNCING_CHAINS_COUNT, + &[RangeSyncType::Finalized.as_str()], + self.finalized_chains.len() as i64, + ); + metrics::set_gauge_vec( + &metrics::SYNCING_CHAINS_COUNT, + &[RangeSyncType::Head.as_str()], + self.head_chains.len() as i64, + ); + } } diff --git a/beacon_node/network/src/sync/range_sync/range.rs b/beacon_node/network/src/sync/range_sync/range.rs index fa06af24956..4213771d483 100644 --- a/beacon_node/network/src/sync/range_sync/range.rs +++ b/beacon_node/network/src/sync/range_sync/range.rs @@ -43,6 +43,7 @@ use super::block_storage::BlockStorage; use super::chain::{BatchId, ChainId, RemoveChain, SyncingChain}; use super::chain_collection::ChainCollection; use super::sync_type::RangeSyncType; +use crate::metrics; use crate::status::ToStatusMessage; use crate::sync::network_context::SyncNetworkContext; use crate::sync::BatchProcessResult; @@ -346,6 +347,12 @@ where } } + metrics::inc_counter_vec_by( + &metrics::SYNCING_CHAINS_DROPPED_BLOCKS, + &[sync_type.as_str()], + chain.pending_blocks() as u64, + ); + network.status_peers(self.beacon_chain.as_ref(), chain.peers()); let status = self.beacon_chain.status_message(); From 7b283c5ddb2d56edd8969ca4ac09327065cb526f Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 16 Jul 2024 04:52:02 +1000 Subject: [PATCH 212/227] Enable the outbound rate limiter by default, and update blobs method quotas (#6093) * Enable the outbound rate limiter by default, and update blobs method quotas. * Lint and book updates. --- .../lighthouse_network/src/rpc/config.rs | 9 ++++++-- beacon_node/src/cli.rs | 21 ++++++++++++------- beacon_node/src/config.rs | 19 ++++++++--------- book/src/help_bn.md | 8 ++----- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/beacon_node/lighthouse_network/src/rpc/config.rs b/beacon_node/lighthouse_network/src/rpc/config.rs index 08b81c7eae5..d17fa112a1b 100644 --- a/beacon_node/lighthouse_network/src/rpc/config.rs +++ b/beacon_node/lighthouse_network/src/rpc/config.rs @@ -103,8 +103,13 @@ impl RateLimiterConfig { pub const DEFAULT_GOODBYE_QUOTA: Quota = Quota::one_every(10); pub const DEFAULT_BLOCKS_BY_RANGE_QUOTA: Quota = Quota::n_every(1024, 10); pub const DEFAULT_BLOCKS_BY_ROOT_QUOTA: Quota = Quota::n_every(128, 10); - pub const DEFAULT_BLOBS_BY_RANGE_QUOTA: Quota = Quota::n_every(768, 10); - pub const DEFAULT_BLOBS_BY_ROOT_QUOTA: Quota = Quota::n_every(128, 10); + // `BlocksByRange` and `BlobsByRange` are sent together during range sync. + // It makes sense for blocks and blobs quotas to be equivalent in terms of the number of blocks: + // 1024 blocks * 6 max blobs per block. + // This doesn't necessarily mean that we are sending this many blobs, because the quotas are + // measured against the maximum request size. + pub const DEFAULT_BLOBS_BY_RANGE_QUOTA: Quota = Quota::n_every(6144, 10); + pub const DEFAULT_BLOBS_BY_ROOT_QUOTA: Quota = Quota::n_every(768, 10); pub const DEFAULT_LIGHT_CLIENT_BOOTSTRAP_QUOTA: Quota = Quota::one_every(10); pub const DEFAULT_LIGHT_CLIENT_OPTIMISTIC_UPDATE_QUOTA: Quota = Quota::one_every(10); pub const DEFAULT_LIGHT_CLIENT_FINALITY_UPDATE_QUOTA: Quota = Quota::one_every(10); diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 40a343a7fe4..c32c5e7ec6f 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -372,17 +372,22 @@ pub fn cli_app() -> Command { .arg( Arg::new("self-limiter") .long("self-limiter") - .help( - "Enables the outbound rate limiter (requests made by this node). \ - Use the self-limiter-protocol flag to set per protocol configurations. \ - If the self rate limiter is enabled and a protocol is not \ - present in the configuration, the quotas used for the inbound rate limiter will be \ - used." - ) + .help("This flag is deprecated and has no effect.") + .hide(true) .action(ArgAction::SetTrue) .help_heading(FLAG_HEADER) .display_order(0) ) + .arg( + Arg::new("disable-self-limiter") + .long("disable-self-limiter") + .help( + "Disables the outbound rate limiter (requests sent by this node)." + ) + .action(ArgAction::SetTrue) + .help_heading(FLAG_HEADER) + .display_order(0) + ) .arg( Arg::new("self-limiter-protocols") .long("self-limiter-protocols") @@ -397,7 +402,7 @@ pub fn cli_app() -> Command { ) .action(ArgAction::Append) .value_delimiter(';') - .requires("self-limiter") + .conflicts_with("disable-self-limiter") .display_order(0) ) .arg( diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index b3c91631c09..f0d02f6c512 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1416,16 +1416,15 @@ pub fn set_network_config( // Light client server config. config.enable_light_client_server = parse_flag(cli_args, "light-client-server"); - // The self limiter is disabled by default. If the `self-limiter` flag is provided - // without the `self-limiter-protocols` flag, the default params will be used. - if parse_flag(cli_args, "self-limiter") { - config.outbound_rate_limiter_config = - if let Some(protocols) = cli_args.get_one::("self-limiter-protocols") { - Some(protocols.parse()?) - } else { - Some(Default::default()) - }; - } + // The self limiter is enabled by default. If the `self-limiter-protocols` flag is not provided, + // the default params will be used. + config.outbound_rate_limiter_config = if parse_flag(cli_args, "disable-self-limiter") { + None + } else if let Some(protocols) = cli_args.get_one::("self-limiter-protocols") { + Some(protocols.parse()?) + } else { + Some(Default::default()) + }; // Proposer-only mode overrides a number of previous configuration parameters. // Specifically, we avoid subscribing to long-lived subnets and wish to maintain a minimal set diff --git a/book/src/help_bn.md b/book/src/help_bn.md index 8bbbd3eb6b7..50484f5ec4f 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -505,6 +505,8 @@ Flags: --disable-quic Disables the quic transport. The node will rely solely on the TCP transport for libp2p connections. + --disable-self-limiter + Disables the outbound rate limiter (requests sent by this node). --disable-upnp Disables UPnP support. Setting this will prevent Lighthouse from attempting to automatically establish external port mappings. @@ -575,12 +577,6 @@ Flags: When present, Lighthouse will forget the payload statuses of any already-imported blocks. This can assist in the recovery from a consensus failure caused by the execution layer. - --self-limiter - Enables the outbound rate limiter (requests made by this node). Use - the self-limiter-protocol flag to set per protocol configurations. If - the self rate limiter is enabled and a protocol is not present in the - configuration, the quotas used for the inbound rate limiter will be - used. --shutdown-after-sync Shutdown beacon node as soon as sync is completed. Backfill sync will not be performed before shutdown. From f290c68c9335403df089ea51d2ff1407b3aeb994 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 15 Jul 2024 12:49:08 -0700 Subject: [PATCH 213/227] Beacon api + validator electra (#5744) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Attestation superstruct changes for EIP 7549 (#5644) * update * experiment * superstruct changes * revert * superstruct changes * fix tests * indexed attestation * indexed attestation superstruct * updated TODOs * `superstruct` the `AttesterSlashing` (#5636) * `superstruct` Attester Fork Variants * Push a little further * Deal with Encode / Decode of AttesterSlashing * not so sure about this.. * Stop Encode/Decode Bounds from Propagating Out * Tons of Changes.. * More Conversions to AttestationRef * Add AsReference trait (#15) * Add AsReference trait * Fix some snafus * Got it Compiling! :D * Got Tests Building * Get beacon chain tests compiling --------- Co-authored-by: Michael Sproul * Merge remote-tracking branch 'upstream/unstable' into electra_attestation_changes * Make EF Tests Fork-Agnostic (#5713) * Finish EF Test Fork Agnostic (#5714) * Superstruct `AggregateAndProof` (#5715) * Upgrade `superstruct` to `0.8.0` * superstruct `AggregateAndProof` * Merge remote-tracking branch 'sigp/unstable' into electra_attestation_changes * cargo fmt * Merge pull request #5726 from realbigsean/electra_attestation_changes Merge unstable into Electra attestation changes * process withdrawals updates * cleanup withdrawals processing * update `process_operations` deposit length check * add apply_deposit changes * add execution layer withdrawal request processing * process deposit receipts * add consolidation processing * update process operations function * exit updates * clean up * update slash_validator * EIP7549 `get_attestation_indices` (#5657) * get attesting indices electra impl * fmt * get tests to pass * fmt * fix some beacon chain tests * fmt * fix slasher test * fmt got me again * fix more tests * fix tests * Some small changes (#5739) * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * cargo fmt (#5740) * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * fix attestation verification * Add new engine api methods * Fix the versioning of v4 requests * Handle new engine api methods in mock EL * Note todo * Fix todos * Add support for electra fields in getPayloadBodies * Add comments for potential versioning confusion * udpates for aggregate attestation endpoint * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * Sketch op pool changes * fix get attesting indices (#5742) * fix get attesting indices * better errors * fix compile * only get committee index once * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Ef test fixes (#5753) * attestation related ef test fixes * delete commented out stuff * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * Fix Aggregation Pool for Electra (#5754) * Fix Aggregation Pool for Electra * Remove Outdated Interface * fix ssz (#5755) * Get `electra_op_pool` up to date (#5756) * fix get attesting indices (#5742) * fix get attesting indices * better errors * fix compile * only get committee index once * Ef test fixes (#5753) * attestation related ef test fixes * delete commented out stuff * Fix Aggregation Pool for Electra (#5754) * Fix Aggregation Pool for Electra * Remove Outdated Interface * fix ssz (#5755) --------- Co-authored-by: realbigsean * Revert "Get `electra_op_pool` up to date (#5756)" (#5757) This reverts commit ab9e58aa3d0e6fe2175a4996a5de710e81152896. * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into electra_op_pool * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * Compute on chain aggregate impl (#5752) * add compute_on_chain_agg impl to op pool changes * fmt * get op pool tests to pass * update beacon api aggregate attestationendpoint * update the naive agg pool interface (#5760) * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * updates after merge * Fix bugs in cross-committee aggregation * Add comment to max cover optimisation * Fix assert * Electra epoch processing * add deposit limit for old deposit queue * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * Merge pull request #5749 from sigp/electra_op_pool Optimise Electra op pool aggregation * don't fail on empty consolidations * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-epoch-proc * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * update committee offset * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * update committee offset * update committee offset * update committee offset * only increment the state deposit index on old deposit flow * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-epoch-proc * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * use correct max eb in epoch cache initialization * drop initiate validator ordering optimization * fix initiate exit for single pass * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * accept new payload v4 in mock el * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * Fix Electra Fork Choice Tests (#5764) * Fix Electra Fork Choice Tests (#5764) * Fix Electra Fork Choice Tests (#5764) * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-epoch-proc * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * Fix Consolidation Sigs & Withdrawals * Merge pull request #5766 from ethDreamer/two_fixes Fix Consolidation Sigs & Withdrawals * Merge branches 'block-processing-electra' and 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-epoch-proc * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * Send unagg attestation based on fork * Fix ser/de * Merge branch 'electra-engine-api' into beacon-api-electra * Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra * cargo fmt * Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra * cargo fmt * Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra * cargo fmt * Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra * cargo fmt * Subscribe to the correct subnets for electra attestations (#5782) * subscribe to the correct att subnets for electra * subscribe to the correct att subnets for electra * cargo fmt * update electra readiness with new endpoints * fix slashing handling * Fix Bug In Block Processing with 0x02 Credentials * Merge remote-tracking branch 'upstream/unstable' * Send unagg attestation based on fork * Publish all aggregates * just one more check bro plz.. * Merge pull request #5832 from ethDreamer/electra_attestation_changes_merge_unstable Merge `unstable` into `electra_attestation_changes` * Merge pull request #5835 from realbigsean/fix-validator-logic Fix validator logic * Merge pull request #5816 from realbigsean/electra-attestation-slashing-handling Electra slashing handling * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-epoch-proc * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * fix: serde rename camle case for execution payload body (#5846) * Merge branch 'electra-engine-api' into beacon-api-electra * Electra attestation changes rm decode impl (#5856) * Remove Crappy Decode impl for Attestation * Remove Inefficient Attestation Decode impl * Implement Schema Upgrade / Downgrade * Update beacon_node/beacon_chain/src/schema_change/migration_schema_v20.rs Co-authored-by: Michael Sproul --------- Co-authored-by: Michael Sproul * Fix failing attestation tests and misc electra attestation cleanup (#5810) * - get attestation related beacon chain tests to pass - observed attestations are now keyed off of data + committee index - rename op pool attestationref to compactattestationref - remove unwraps in agg pool and use options instead - cherry pick some changes from ef-tests-electra * cargo fmt * fix failing test * Revert dockerfile changes * make committee_index return option * function args shouldnt be a ref to attestation ref * fmt * fix dup imports --------- Co-authored-by: realbigsean * fix some todos (#5817) * Merge branch 'unstable' of https://github.com/sigp/lighthouse into electra_attestation_changes * add consolidations to merkle calc for inclusion proof * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-epoch-proc * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * Remove Duplicate KZG Commitment Merkle Proof Code (#5874) * Remove Duplicate KZG Commitment Merkle Proof Code * s/tree_lists/fields/ * Merge branch 'unstable' of https://github.com/sigp/lighthouse into electra_attestation_changes * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-epoch-proc * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * fix compile * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-epoch-proc * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * Fix slasher tests (#5906) * Fix electra tests * Add electra attestations to double vote tests * Update superstruct to 0.8 * Merge remote-tracking branch 'origin/unstable' into electra_attestation_changes * Small cleanup in slasher tests * Clean up Electra observed aggregates (#5929) * Use consistent key in observed_attestations * Remove unwraps from observed aggregates * Merge branch 'unstable' of https://github.com/sigp/lighthouse into electra_attestation_changes * De-dup attestation constructor logic * Remove unwraps in Attestation construction * Dedup match_attestation_data * Remove outdated TODO * Use ForkName Ord in fork-choice tests * Use ForkName Ord in BeaconBlockBody * Make to_electra not fallible * Remove TestRandom impl for IndexedAttestation * Remove IndexedAttestation faulty Decode impl * Drop TestRandom impl * Add PendingAttestationInElectra * Indexed att on disk (#35) * indexed att on disk * fix lints * Update slasher/src/migrate.rs Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com> --------- Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com> * add electra fork enabled fn to ForkName impl (#36) * add electra fork enabled fn to ForkName impl * remove inadvertent file * Update common/eth2/src/types.rs Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com> * Dedup attestation constructor logic in attester cache * Use if let Ok for committee_bits * Dedup Attestation constructor code * Diff reduction in tests * Fix beacon_chain tests * Diff reduction * Use Ord for ForkName in pubsub * Resolve into_attestation_and_indices todo * Remove stale TODO * Fix beacon_chain tests * Test spec invariant * Use electra_enabled in pubsub * Remove get_indexed_attestation_from_signed_aggregate * Use ok_or instead of if let else * committees are sorted * remove dup method `get_indexed_attestation_from_committees` * Merge pull request #5940 from dapplion/electra_attestation_changes_lionreview Electra attestations #5712 review * update default persisted op pool deserialization * ensure aggregate and proof uses serde untagged on ref * Fork aware ssz static attestation tests * Electra attestation changes from Lions review (#5971) * dedup/cleanup and remove unneeded hashset use * remove irrelevant TODOs * Merge branch 'unstable' of https://github.com/sigp/lighthouse into electra_attestation_changes * Merge branch 'electra_attestation_changes' of https://github.com/sigp/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-epoch-proc * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * Fix Compilation Break * Merge pull request #5973 from ethDreamer/beacon-api-electra Fix Compilation Break * Electra attestation changes sean review (#5972) * instantiate empty bitlist in unreachable code * clean up error conversion * fork enabled bool cleanup * remove a couple todos * return bools instead of options in `aggregate` and use the result * delete commented out code * use map macros in simple transformations * remove signers_disjoint_from * get ef tests compiling * get ef tests compiling * update intentionally excluded files * Avoid changing slasher schema for Electra * Delete slasher schema v4 * Fix clippy * Fix compilation of beacon_chain tests * Update database.rs * Update per_block_processing.rs * Add electra lightclient types * Update slasher/src/database.rs * fix imports * Merge pull request #5980 from dapplion/electra-lightclient Add electra lightclient types * Merge pull request #5975 from michaelsproul/electra-slasher-no-migration Avoid changing slasher schema for Electra * Update beacon_node/beacon_chain/src/attestation_verification.rs * Update beacon_node/beacon_chain/src/attestation_verification.rs * Merge branch 'unstable' of https://github.com/sigp/lighthouse into electra_attestation_changes * Merge branch 'electra_attestation_changes' of https://github.com/realbigsean/lighthouse into block-processing-electra * Merge branch 'block-processing-electra' of https://github.com/sigp/lighthouse into electra-epoch-proc * Merge branch 'electra-epoch-proc' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * The great renaming receipt -> request * Address some more review comments * Merge branch 'unstable' of https://github.com/sigp/lighthouse into electra-engine-api * Update beacon_node/beacon_chain/src/electra_readiness.rs * Update consensus/types/src/chain_spec.rs * update GET requests * update POST requests * add client updates and test updates * Merge branch 'unstable' of https://github.com/sigp/lighthouse into electra-engine-api * Merge branch 'electra-engine-api' of https://github.com/sigp/lighthouse into beacon-api-electra * compile after merge * unwrap -> unwrap_err * self review * fix tests * convert op pool messages to electra in electra * remove methods to post without content header * filter instead of convert --- beacon_node/beacon_chain/src/eth1_chain.rs | 12 +- beacon_node/http_api/src/lib.rs | 202 +++++++--- beacon_node/http_api/tests/fork_tests.rs | 13 +- .../http_api/tests/interactive_tests.rs | 3 +- beacon_node/http_api/tests/tests.rs | 362 +++++++++++++++--- common/eth2/src/lib.rs | 218 +++++++++-- common/eth2/src/types.rs | 2 + consensus/types/src/attestation.rs | 48 ++- consensus/types/src/attester_slashing.rs | 23 ++ validator_client/src/attestation_service.rs | 108 ++++-- 10 files changed, 817 insertions(+), 174 deletions(-) diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index ee50e3b3843..62aad558eeb 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -546,12 +546,20 @@ impl Eth1ChainBackend for CachingEth1Backend { state.eth1_data().deposit_count }; - match deposit_index.cmp(&deposit_count) { + // [New in Electra:EIP6110] + let deposit_index_limit = + if let Ok(deposit_receipts_start_index) = state.deposit_requests_start_index() { + std::cmp::min(deposit_count, deposit_receipts_start_index) + } else { + deposit_count + }; + + match deposit_index.cmp(&deposit_index_limit) { Ordering::Greater => Err(Error::DepositIndexTooHigh), Ordering::Equal => Ok(vec![]), Ordering::Less => { let next = deposit_index; - let last = std::cmp::min(deposit_count, next + E::MaxDeposits::to_u64()); + let last = std::cmp::min(deposit_index_limit, next + E::MaxDeposits::to_u64()); self.core .deposits() diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 7c8f64a722c..2d017d65391 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -31,6 +31,7 @@ mod validators; mod version; use crate::produce_block::{produce_blinded_block_v2, produce_block_v2, produce_block_v3}; +use crate::version::fork_versioned_response; use beacon_chain::{ attestation_verification::VerifiedAttestation, observed_operations::ObservationOutcome, validator_monitor::timestamp_now, AttestationError as AttnError, BeaconChain, BeaconChainError, @@ -256,12 +257,15 @@ pub fn prometheus_metrics() -> warp::filters::log::Log( ); // GET beacon/blocks/{block_id}/attestations - let get_beacon_block_attestations = beacon_blocks_path_v1 + let get_beacon_block_attestations = beacon_blocks_path_any .clone() .and(warp::path("attestations")) .and(warp::path::end()) .then( - |block_id: BlockId, + |endpoint_version: EndpointVersion, + block_id: BlockId, task_spawner: TaskSpawner, chain: Arc>| { - task_spawner.blocking_json_task(Priority::P1, move || { + task_spawner.blocking_response_task(Priority::P1, move || { let (block, execution_optimistic, finalized) = block_id.blinded_block(&chain)?; - Ok(api_types::GenericResponse::from( - block - .message() - .body() - .attestations() - .map(|att| att.clone_as_attestation()) - .collect::>(), - ) - .add_execution_optimistic_finalized(execution_optimistic, finalized)) + let fork_name = block + .fork_name(&chain.spec) + .map_err(inconsistent_fork_rejection)?; + let atts = block + .message() + .body() + .attestations() + .map(|att| att.clone_as_attestation()) + .collect::>(); + let res = execution_optimistic_finalized_fork_versioned_response( + endpoint_version, + fork_name, + execution_optimistic, + finalized, + &atts, + )?; + Ok(add_consensus_version_header( + warp::reply::json(&res).into_response(), + fork_name, + )) }) }, ); @@ -1750,8 +1766,14 @@ pub fn serve( .and(task_spawner_filter.clone()) .and(chain_filter.clone()); + let beacon_pool_path_any = any_version + .and(warp::path("beacon")) + .and(warp::path("pool")) + .and(task_spawner_filter.clone()) + .and(chain_filter.clone()); + // POST beacon/pool/attestations - let post_beacon_pool_attestations = beacon_pool_path + let post_beacon_pool_attestations = beacon_pool_path_any .clone() .and(warp::path("attestations")) .and(warp::path::end()) @@ -1760,7 +1782,11 @@ pub fn serve( .and(reprocess_send_filter) .and(log_filter.clone()) .then( - |task_spawner: TaskSpawner, + // V1 and V2 are identical except V2 has a consensus version header in the request. + // We only require this header for SSZ deserialization, which isn't supported for + // this endpoint presently. + |_endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, chain: Arc>, attestations: Vec>, network_tx: UnboundedSender>, @@ -1781,16 +1807,17 @@ pub fn serve( ); // GET beacon/pool/attestations?committee_index,slot - let get_beacon_pool_attestations = beacon_pool_path + let get_beacon_pool_attestations = beacon_pool_path_any .clone() .and(warp::path("attestations")) .and(warp::path::end()) .and(warp::query::()) .then( - |task_spawner: TaskSpawner, + |endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, chain: Arc>, query: api_types::AttestationPoolQuery| { - task_spawner.blocking_json_task(Priority::P1, move || { + task_spawner.blocking_response_task(Priority::P1, move || { let query_filter = |data: &AttestationData| { query.slot.map_or(true, |slot| slot == data.slot) && query @@ -1807,20 +1834,48 @@ pub fn serve( .filter(|&att| query_filter(att.data())) .cloned(), ); - Ok(api_types::GenericResponse::from(attestations)) + // Use the current slot to find the fork version, and convert all messages to the + // current fork's format. This is to ensure consistent message types matching + // `Eth-Consensus-Version`. + let current_slot = + chain + .slot_clock + .now() + .ok_or(warp_utils::reject::custom_server_error( + "unable to read slot clock".to_string(), + ))?; + let fork_name = chain.spec.fork_name_at_slot::(current_slot); + let attestations = attestations + .into_iter() + .filter(|att| { + (fork_name.electra_enabled() && matches!(att, Attestation::Electra(_))) + || (!fork_name.electra_enabled() + && matches!(att, Attestation::Base(_))) + }) + .collect::>(); + + let res = fork_versioned_response(endpoint_version, fork_name, &attestations)?; + Ok(add_consensus_version_header( + warp::reply::json(&res).into_response(), + fork_name, + )) }) }, ); // POST beacon/pool/attester_slashings - let post_beacon_pool_attester_slashings = beacon_pool_path + let post_beacon_pool_attester_slashings = beacon_pool_path_any .clone() .and(warp::path("attester_slashings")) .and(warp::path::end()) .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .then( - |task_spawner: TaskSpawner, + // V1 and V2 are identical except V2 has a consensus version header in the request. + // We only require this header for SSZ deserialization, which isn't supported for + // this endpoint presently. + |_endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, chain: Arc>, slashing: AttesterSlashing, network_tx: UnboundedSender>| { @@ -1857,18 +1912,45 @@ pub fn serve( ); // GET beacon/pool/attester_slashings - let get_beacon_pool_attester_slashings = beacon_pool_path - .clone() - .and(warp::path("attester_slashings")) - .and(warp::path::end()) - .then( - |task_spawner: TaskSpawner, chain: Arc>| { - task_spawner.blocking_json_task(Priority::P1, move || { - let attestations = chain.op_pool.get_all_attester_slashings(); - Ok(api_types::GenericResponse::from(attestations)) - }) - }, - ); + let get_beacon_pool_attester_slashings = + beacon_pool_path_any + .clone() + .and(warp::path("attester_slashings")) + .and(warp::path::end()) + .then( + |endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_response_task(Priority::P1, move || { + let slashings = chain.op_pool.get_all_attester_slashings(); + + // Use the current slot to find the fork version, and convert all messages to the + // current fork's format. This is to ensure consistent message types matching + // `Eth-Consensus-Version`. + let current_slot = chain.slot_clock.now().ok_or( + warp_utils::reject::custom_server_error( + "unable to read slot clock".to_string(), + ), + )?; + let fork_name = chain.spec.fork_name_at_slot::(current_slot); + let slashings = slashings + .into_iter() + .filter(|slashing| { + (fork_name.electra_enabled() + && matches!(slashing, AttesterSlashing::Electra(_))) + || (!fork_name.electra_enabled() + && matches!(slashing, AttesterSlashing::Base(_))) + }) + .collect::>(); + + let res = fork_versioned_response(endpoint_version, fork_name, &slashings)?; + Ok(add_consensus_version_header( + warp::reply::json(&res).into_response(), + fork_name, + )) + }) + }, + ); // POST beacon/pool/proposer_slashings let post_beacon_pool_proposer_slashings = beacon_pool_path @@ -3175,7 +3257,7 @@ pub fn serve( ); // GET validator/aggregate_attestation?attestation_data_root,slot - let get_validator_aggregate_attestation = eth_v1 + let get_validator_aggregate_attestation = any_version .and(warp::path("validator")) .and(warp::path("aggregate_attestation")) .and(warp::path::end()) @@ -3184,29 +3266,45 @@ pub fn serve( .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( - |query: api_types::ValidatorAggregateAttestationQuery, + |endpoint_version: EndpointVersion, + query: api_types::ValidatorAggregateAttestationQuery, not_synced_filter: Result<(), Rejection>, task_spawner: TaskSpawner, chain: Arc>| { task_spawner.blocking_json_task(Priority::P0, move || { not_synced_filter?; - chain - .get_pre_electra_aggregated_attestation_by_slot_and_root( + let res = if endpoint_version == V2 { + let Some(committee_index) = query.committee_index else { + return Err(warp_utils::reject::custom_bad_request( + "missing committee index".to_string(), + )); + }; + chain.get_aggregated_attestation_electra( query.slot, &query.attestation_data_root, + committee_index, ) - .map_err(|e| { - warp_utils::reject::custom_bad_request(format!( - "unable to fetch aggregate: {:?}", - e - )) - })? - .map(api_types::GenericResponse::from) - .ok_or_else(|| { - warp_utils::reject::custom_not_found( - "no matching aggregate found".to_string(), - ) - }) + } else if endpoint_version == V1 { + // Do nothing + chain.get_pre_electra_aggregated_attestation_by_slot_and_root( + query.slot, + &query.attestation_data_root, + ) + } else { + return Err(unsupported_version_rejection(endpoint_version)); + }; + res.map_err(|e| { + warp_utils::reject::custom_bad_request(format!( + "unable to fetch aggregate: {:?}", + e + )) + })? + .map(api_types::GenericResponse::from) + .ok_or_else(|| { + warp_utils::reject::custom_not_found( + "no matching aggregate found".to_string(), + ) + }) }) }, ); @@ -3302,7 +3400,7 @@ pub fn serve( ); // POST validator/aggregate_and_proofs - let post_validator_aggregate_and_proofs = eth_v1 + let post_validator_aggregate_and_proofs = any_version .and(warp::path("validator")) .and(warp::path("aggregate_and_proofs")) .and(warp::path::end()) @@ -3313,7 +3411,11 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |not_synced_filter: Result<(), Rejection>, + // V1 and V2 are identical except V2 has a consensus version header in the request. + // We only require this header for SSZ deserialization, which isn't supported for + // this endpoint presently. + |_endpoint_version: EndpointVersion, + not_synced_filter: Result<(), Rejection>, task_spawner: TaskSpawner, chain: Arc>, aggregates: Vec>, diff --git a/beacon_node/http_api/tests/fork_tests.rs b/beacon_node/http_api/tests/fork_tests.rs index 1e20280da11..db8a0ab2b54 100644 --- a/beacon_node/http_api/tests/fork_tests.rs +++ b/beacon_node/http_api/tests/fork_tests.rs @@ -150,8 +150,13 @@ async fn attestations_across_fork_with_skip_slots() { .collect::>(); assert!(!unaggregated_attestations.is_empty()); + let fork_name = harness.spec.fork_name_at_slot::(fork_slot); client - .post_beacon_pool_attestations(&unaggregated_attestations) + .post_beacon_pool_attestations_v1(&unaggregated_attestations) + .await + .unwrap(); + client + .post_beacon_pool_attestations_v2(&unaggregated_attestations, fork_name) .await .unwrap(); @@ -162,7 +167,11 @@ async fn attestations_across_fork_with_skip_slots() { assert!(!signed_aggregates.is_empty()); client - .post_validator_aggregate_and_proof(&signed_aggregates) + .post_validator_aggregate_and_proof_v1(&signed_aggregates) + .await + .unwrap(); + client + .post_validator_aggregate_and_proof_v2(&signed_aggregates, fork_name) .await .unwrap(); } diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 14673d23e12..2f417cf7ba5 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -893,9 +893,10 @@ async fn queue_attestations_from_http() { .flat_map(|attestations| attestations.into_iter().map(|(att, _subnet)| att)) .collect::>(); + let fork_name = tester.harness.spec.fork_name_at_slot::(attestation_slot); let attestation_future = tokio::spawn(async move { client - .post_beacon_pool_attestations(&attestations) + .post_beacon_pool_attestations_v2(&attestations, fork_name) .await .expect("attestations should be processed successfully") }); diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index cb4ce346827..f511f25d327 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -1668,7 +1668,7 @@ impl ApiTester { for block_id in self.interesting_block_ids() { let result = self .client - .get_beacon_blocks_attestations(block_id.0) + .get_beacon_blocks_attestations_v2(block_id.0) .await .unwrap() .map(|res| res.data); @@ -1699,9 +1699,9 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attestations_valid(mut self) -> Self { + pub async fn test_post_beacon_pool_attestations_valid_v1(mut self) -> Self { self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -1713,7 +1713,25 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attestations_invalid(mut self) -> Self { + pub async fn test_post_beacon_pool_attestations_valid_v2(mut self) -> Self { + let fork_name = self + .attestations + .first() + .map(|att| self.chain.spec.fork_name_at_slot::(att.data().slot)) + .unwrap(); + self.client + .post_beacon_pool_attestations_v2(self.attestations.as_slice(), fork_name) + .await + .unwrap(); + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "valid attestation should be sent to network" + ); + + self + } + + pub async fn test_post_beacon_pool_attestations_invalid_v1(mut self) -> Self { let mut attestations = Vec::new(); for attestation in &self.attestations { let mut invalid_attestation = attestation.clone(); @@ -1726,7 +1744,7 @@ impl ApiTester { let err = self .client - .post_beacon_pool_attestations(attestations.as_slice()) + .post_beacon_pool_attestations_v1(attestations.as_slice()) .await .unwrap_err(); @@ -1749,6 +1767,48 @@ impl ApiTester { self } + pub async fn test_post_beacon_pool_attestations_invalid_v2(mut self) -> Self { + let mut attestations = Vec::new(); + for attestation in &self.attestations { + let mut invalid_attestation = attestation.clone(); + invalid_attestation.data_mut().slot += 1; + + // add both to ensure we only fail on invalid attestations + attestations.push(attestation.clone()); + attestations.push(invalid_attestation); + } + + let fork_name = self + .attestations + .first() + .map(|att| self.chain.spec.fork_name_at_slot::(att.data().slot)) + .unwrap(); + + let err_v2 = self + .client + .post_beacon_pool_attestations_v2(attestations.as_slice(), fork_name) + .await + .unwrap_err(); + + match err_v2 { + Error::ServerIndexedMessage(IndexedErrorMessage { + code, + message: _, + failures, + }) => { + assert_eq!(code, 400); + assert_eq!(failures.len(), self.attestations.len()); + } + _ => panic!("query did not fail correctly"), + } + + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "if some attestations are valid, we should send them to the network" + ); + + self + } pub async fn test_get_beacon_light_client_bootstrap(self) -> Self { let block_id = BlockId(CoreBlockId::Finalized); @@ -1812,7 +1872,7 @@ impl ApiTester { pub async fn test_get_beacon_pool_attestations(self) -> Self { let result = self .client - .get_beacon_pool_attestations(None, None) + .get_beacon_pool_attestations_v1(None, None) .await .unwrap() .data; @@ -1822,12 +1882,38 @@ impl ApiTester { assert_eq!(result, expected); + let result = self + .client + .get_beacon_pool_attestations_v2(None, None) + .await + .unwrap() + .data; + assert_eq!(result, expected); + + self + } + + pub async fn test_post_beacon_pool_attester_slashings_valid_v1(mut self) -> Self { + self.client + .post_beacon_pool_attester_slashings_v1(&self.attester_slashing) + .await + .unwrap(); + + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "valid attester slashing should be sent to network" + ); + self } - pub async fn test_post_beacon_pool_attester_slashings_valid(mut self) -> Self { + pub async fn test_post_beacon_pool_attester_slashings_valid_v2(mut self) -> Self { + let fork_name = self + .chain + .spec + .fork_name_at_slot::(self.attester_slashing.attestation_1().data().slot); self.client - .post_beacon_pool_attester_slashings(&self.attester_slashing) + .post_beacon_pool_attester_slashings_v2(&self.attester_slashing, fork_name) .await .unwrap(); @@ -1839,7 +1925,31 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attester_slashings_invalid(mut self) -> Self { + pub async fn test_post_beacon_pool_attester_slashings_invalid_v1(mut self) -> Self { + let mut slashing = self.attester_slashing.clone(); + match &mut slashing { + AttesterSlashing::Base(ref mut slashing) => { + slashing.attestation_1.data.slot += 1; + } + AttesterSlashing::Electra(ref mut slashing) => { + slashing.attestation_1.data.slot += 1; + } + } + + self.client + .post_beacon_pool_attester_slashings_v1(&slashing) + .await + .unwrap_err(); + + assert!( + self.network_rx.network_recv.recv().now_or_never().is_none(), + "invalid attester slashing should not be sent to network" + ); + + self + } + + pub async fn test_post_beacon_pool_attester_slashings_invalid_v2(mut self) -> Self { let mut slashing = self.attester_slashing.clone(); match &mut slashing { AttesterSlashing::Base(ref mut slashing) => { @@ -1850,8 +1960,12 @@ impl ApiTester { } } + let fork_name = self + .chain + .spec + .fork_name_at_slot::(self.attester_slashing.attestation_1().data().slot); self.client - .post_beacon_pool_attester_slashings(&slashing) + .post_beacon_pool_attester_slashings_v2(&slashing, fork_name) .await .unwrap_err(); @@ -1866,7 +1980,7 @@ impl ApiTester { pub async fn test_get_beacon_pool_attester_slashings(self) -> Self { let result = self .client - .get_beacon_pool_attester_slashings() + .get_beacon_pool_attester_slashings_v1() .await .unwrap() .data; @@ -1875,6 +1989,14 @@ impl ApiTester { assert_eq!(result, expected); + let result = self + .client + .get_beacon_pool_attester_slashings_v2() + .await + .unwrap() + .data; + assert_eq!(result, expected); + self } @@ -3233,30 +3355,52 @@ impl ApiTester { } pub async fn test_get_validator_aggregate_attestation(self) -> Self { - let attestation = self + if self .chain - .head_beacon_block() - .message() - .body() - .attestations() - .next() - .unwrap() - .clone_as_attestation(); - - let result = self - .client - .get_validator_aggregate_attestation( - attestation.data().slot, - attestation.data().tree_hash_root(), - ) - .await - .unwrap() - .unwrap() - .data; + .spec + .fork_name_at_slot::(self.chain.slot().unwrap()) + .electra_enabled() + { + for attestation in self.chain.naive_aggregation_pool.read().iter() { + let result = self + .client + .get_validator_aggregate_attestation_v2( + attestation.data().slot, + attestation.data().tree_hash_root(), + attestation.committee_index().unwrap(), + ) + .await + .unwrap() + .unwrap() + .data; + let expected = attestation; - let expected = attestation; + assert_eq!(&result, expected); + } + } else { + let attestation = self + .chain + .head_beacon_block() + .message() + .body() + .attestations() + .next() + .unwrap() + .clone_as_attestation(); + let result = self + .client + .get_validator_aggregate_attestation_v1( + attestation.data().slot, + attestation.data().tree_hash_root(), + ) + .await + .unwrap() + .unwrap() + .data; + let expected = attestation; - assert_eq!(result, expected); + assert_eq!(result, expected); + } self } @@ -3355,11 +3499,11 @@ impl ApiTester { ) } - pub async fn test_get_validator_aggregate_and_proofs_valid(mut self) -> Self { + pub async fn test_get_validator_aggregate_and_proofs_valid_v1(mut self) -> Self { let aggregate = self.get_aggregate().await; self.client - .post_validator_aggregate_and_proof::(&[aggregate]) + .post_validator_aggregate_and_proof_v1::(&[aggregate]) .await .unwrap(); @@ -3368,7 +3512,7 @@ impl ApiTester { self } - pub async fn test_get_validator_aggregate_and_proofs_invalid(mut self) -> Self { + pub async fn test_get_validator_aggregate_and_proofs_invalid_v1(mut self) -> Self { let mut aggregate = self.get_aggregate().await; match &mut aggregate { SignedAggregateAndProof::Base(ref mut aggregate) => { @@ -3380,7 +3524,7 @@ impl ApiTester { } self.client - .post_validator_aggregate_and_proof::(&[aggregate]) + .post_validator_aggregate_and_proof_v1::(&[aggregate.clone()]) .await .unwrap_err(); @@ -3389,6 +3533,46 @@ impl ApiTester { self } + pub async fn test_get_validator_aggregate_and_proofs_valid_v2(mut self) -> Self { + let aggregate = self.get_aggregate().await; + let fork_name = self + .chain + .spec + .fork_name_at_slot::(aggregate.message().aggregate().data().slot); + self.client + .post_validator_aggregate_and_proof_v2::(&[aggregate], fork_name) + .await + .unwrap(); + + assert!(self.network_rx.network_recv.recv().await.is_some()); + + self + } + + pub async fn test_get_validator_aggregate_and_proofs_invalid_v2(mut self) -> Self { + let mut aggregate = self.get_aggregate().await; + match &mut aggregate { + SignedAggregateAndProof::Base(ref mut aggregate) => { + aggregate.message.aggregate.data.slot += 1; + } + SignedAggregateAndProof::Electra(ref mut aggregate) => { + aggregate.message.aggregate.data.slot += 1; + } + } + + let fork_name = self + .chain + .spec + .fork_name_at_slot::(aggregate.message().aggregate().data().slot); + self.client + .post_validator_aggregate_and_proof_v2::(&[aggregate], fork_name) + .await + .unwrap_err(); + assert!(self.network_rx.network_recv.recv().now_or_never().is_none()); + + self + } + pub async fn test_get_validator_beacon_committee_subscriptions(mut self) -> Self { let subscription = BeaconCommitteeSubscription { validator_index: 0, @@ -3484,7 +3668,7 @@ impl ApiTester { pub async fn test_post_validator_register_validator_slashed(self) -> Self { // slash a validator self.client - .post_beacon_pool_attester_slashings(&self.attester_slashing) + .post_beacon_pool_attester_slashings_v1(&self.attester_slashing) .await .unwrap(); @@ -3597,7 +3781,7 @@ impl ApiTester { // Attest to the current slot self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -5237,7 +5421,7 @@ impl ApiTester { // Attest to the current slot self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -5292,7 +5476,7 @@ impl ApiTester { let expected_attestation_len = self.attestations.len(); self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -5801,34 +5985,66 @@ async fn post_beacon_blocks_duplicate() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attestations_valid() { +async fn beacon_pools_post_attestations_valid_v1() { + ApiTester::new() + .await + .test_post_beacon_pool_attestations_valid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attestations_invalid_v1() { + ApiTester::new() + .await + .test_post_beacon_pool_attestations_invalid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attestations_valid_v2() { ApiTester::new() .await - .test_post_beacon_pool_attestations_valid() + .test_post_beacon_pool_attestations_valid_v2() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attestations_invalid() { +async fn beacon_pools_post_attestations_invalid_v2() { ApiTester::new() .await - .test_post_beacon_pool_attestations_invalid() + .test_post_beacon_pool_attestations_invalid_v2() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attester_slashings_valid() { +async fn beacon_pools_post_attester_slashings_valid_v1() { ApiTester::new() .await - .test_post_beacon_pool_attester_slashings_valid() + .test_post_beacon_pool_attester_slashings_valid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attester_slashings_invalid() { +async fn beacon_pools_post_attester_slashings_invalid_v1() { ApiTester::new() .await - .test_post_beacon_pool_attester_slashings_invalid() + .test_post_beacon_pool_attester_slashings_invalid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attester_slashings_valid_v2() { + ApiTester::new() + .await + .test_post_beacon_pool_attester_slashings_valid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attester_slashings_invalid_v2() { + ApiTester::new() + .await + .test_post_beacon_pool_attester_slashings_invalid_v2() .await; } @@ -6156,36 +6372,70 @@ async fn get_validator_aggregate_attestation_with_skip_slots() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_valid() { +async fn get_validator_aggregate_and_proofs_valid_v1() { + ApiTester::new() + .await + .test_get_validator_aggregate_and_proofs_valid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_valid_with_skip_slots_v1() { + ApiTester::new() + .await + .skip_slots(E::slots_per_epoch() * 2) + .test_get_validator_aggregate_and_proofs_valid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_valid_v2() { + ApiTester::new() + .await + .test_get_validator_aggregate_and_proofs_valid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_valid_with_skip_slots_v2() { + ApiTester::new() + .await + .skip_slots(E::slots_per_epoch() * 2) + .test_get_validator_aggregate_and_proofs_valid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_invalid_v1() { ApiTester::new() .await - .test_get_validator_aggregate_and_proofs_valid() + .test_get_validator_aggregate_and_proofs_invalid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_valid_with_skip_slots() { +async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots_v1() { ApiTester::new() .await .skip_slots(E::slots_per_epoch() * 2) - .test_get_validator_aggregate_and_proofs_valid() + .test_get_validator_aggregate_and_proofs_invalid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_invalid() { +async fn get_validator_aggregate_and_proofs_invalid_v2() { ApiTester::new() .await - .test_get_validator_aggregate_and_proofs_invalid() + .test_get_validator_aggregate_and_proofs_invalid_v2() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots() { +async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots_v2() { ApiTester::new() .await .skip_slots(E::slots_per_epoch() * 2) - .test_get_validator_aggregate_and_proofs_invalid() + .test_get_validator_aggregate_and_proofs_invalid_v2() .await; } diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 5a51aaec5a2..6d000f576f9 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -346,6 +346,19 @@ impl BeaconNodeHttpClient { Ok(()) } + /// Perform a HTTP POST request with a custom timeout and consensus header. + async fn post_with_timeout_and_consensus_header( + &self, + url: U, + body: &T, + timeout: Duration, + fork_name: ForkName, + ) -> Result<(), Error> { + self.post_generic_with_consensus_version(url, body, Some(timeout), fork_name) + .await?; + Ok(()) + } + /// Perform a HTTP POST request with a custom timeout, returning a JSON response. async fn post_with_timeout_and_response( &self, @@ -376,25 +389,6 @@ impl BeaconNodeHttpClient { ok_or_error(response).await } - /// Generic POST function supporting arbitrary responses and timeouts. - /// Does not include Content-Type application/json in the request header. - async fn post_generic_json_without_content_type_header( - &self, - url: U, - body: &T, - timeout: Option, - ) -> Result { - let mut builder = self.client.post(url); - if let Some(timeout) = timeout { - builder = builder.timeout(timeout); - } - - let serialized_body = serde_json::to_vec(body).map_err(Error::InvalidJson)?; - - let response = builder.body(serialized_body).send().await?; - ok_or_error(response).await - } - /// Generic POST function supporting arbitrary responses and timeouts. async fn post_generic_with_consensus_version( &self, @@ -1228,10 +1222,10 @@ impl BeaconNodeHttpClient { self.get_opt(path).await } - /// `GET beacon/blocks/{block_id}/attestations` + /// `GET v1/beacon/blocks/{block_id}/attestations` /// /// Returns `Ok(None)` on a 404 error. - pub async fn get_beacon_blocks_attestations( + pub async fn get_beacon_blocks_attestations_v1( &self, block_id: BlockId, ) -> Result>>>, Error> { @@ -1247,8 +1241,28 @@ impl BeaconNodeHttpClient { self.get_opt(path).await } - /// `POST beacon/pool/attestations` - pub async fn post_beacon_pool_attestations( + /// `GET v2/beacon/blocks/{block_id}/attestations` + /// + /// Returns `Ok(None)` on a 404 error. + pub async fn get_beacon_blocks_attestations_v2( + &self, + block_id: BlockId, + ) -> Result>>>, Error> + { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("blocks") + .push(&block_id.to_string()) + .push("attestations"); + + self.get_opt(path).await + } + + /// `POST v1/beacon/pool/attestations` + pub async fn post_beacon_pool_attestations_v1( &self, attestations: &[Attestation], ) -> Result<(), Error> { @@ -1266,8 +1280,33 @@ impl BeaconNodeHttpClient { Ok(()) } - /// `GET beacon/pool/attestations?slot,committee_index` - pub async fn get_beacon_pool_attestations( + /// `POST v2/beacon/pool/attestations` + pub async fn post_beacon_pool_attestations_v2( + &self, + attestations: &[Attestation], + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attestations"); + + self.post_with_timeout_and_consensus_header( + path, + &attestations, + self.timeouts.attestation, + fork_name, + ) + .await?; + + Ok(()) + } + + /// `GET v1/beacon/pool/attestations?slot,committee_index` + pub async fn get_beacon_pool_attestations_v1( &self, slot: Option, committee_index: Option, @@ -1293,8 +1332,35 @@ impl BeaconNodeHttpClient { self.get(path).await } - /// `POST beacon/pool/attester_slashings` - pub async fn post_beacon_pool_attester_slashings( + /// `GET v2/beacon/pool/attestations?slot,committee_index` + pub async fn get_beacon_pool_attestations_v2( + &self, + slot: Option, + committee_index: Option, + ) -> Result>>, Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attestations"); + + if let Some(slot) = slot { + path.query_pairs_mut() + .append_pair("slot", &slot.to_string()); + } + + if let Some(index) = committee_index { + path.query_pairs_mut() + .append_pair("committee_index", &index.to_string()); + } + + self.get(path).await + } + + /// `POST v1/beacon/pool/attester_slashings` + pub async fn post_beacon_pool_attester_slashings_v1( &self, slashing: &AttesterSlashing, ) -> Result<(), Error> { @@ -1306,14 +1372,33 @@ impl BeaconNodeHttpClient { .push("pool") .push("attester_slashings"); - self.post_generic_json_without_content_type_header(path, slashing, None) + self.post_generic(path, slashing, None).await?; + + Ok(()) + } + + /// `POST v2/beacon/pool/attester_slashings` + pub async fn post_beacon_pool_attester_slashings_v2( + &self, + slashing: &AttesterSlashing, + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attester_slashings"); + + self.post_generic_with_consensus_version(path, slashing, None, fork_name) .await?; Ok(()) } - /// `GET beacon/pool/attester_slashings` - pub async fn get_beacon_pool_attester_slashings( + /// `GET v1/beacon/pool/attester_slashings` + pub async fn get_beacon_pool_attester_slashings_v1( &self, ) -> Result>>, Error> { let mut path = self.eth_path(V1)?; @@ -1327,6 +1412,21 @@ impl BeaconNodeHttpClient { self.get(path).await } + /// `GET v2/beacon/pool/attester_slashings` + pub async fn get_beacon_pool_attester_slashings_v2( + &self, + ) -> Result>>, Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attester_slashings"); + + self.get(path).await + } + /// `POST beacon/pool/proposer_slashings` pub async fn post_beacon_pool_proposer_slashings( &self, @@ -2216,8 +2316,8 @@ impl BeaconNodeHttpClient { self.get_with_timeout(path, self.timeouts.attestation).await } - /// `GET validator/aggregate_attestation?slot,attestation_data_root` - pub async fn get_validator_aggregate_attestation( + /// `GET v1/validator/aggregate_attestation?slot,attestation_data_root` + pub async fn get_validator_aggregate_attestation_v1( &self, slot: Slot, attestation_data_root: Hash256, @@ -2240,6 +2340,32 @@ impl BeaconNodeHttpClient { .await } + /// `GET v2/validator/aggregate_attestation?slot,attestation_data_root,committee_index` + pub async fn get_validator_aggregate_attestation_v2( + &self, + slot: Slot, + attestation_data_root: Hash256, + committee_index: CommitteeIndex, + ) -> Result>>, Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("validator") + .push("aggregate_attestation"); + + path.query_pairs_mut() + .append_pair("slot", &slot.to_string()) + .append_pair( + "attestation_data_root", + &format!("{:?}", attestation_data_root), + ) + .append_pair("committee_index", &committee_index.to_string()); + + self.get_opt_with_timeout(path, self.timeouts.attestation) + .await + } + /// `GET validator/sync_committee_contribution` pub async fn get_validator_sync_committee_contribution( &self, @@ -2335,8 +2461,8 @@ impl BeaconNodeHttpClient { .await } - /// `POST validator/aggregate_and_proofs` - pub async fn post_validator_aggregate_and_proof( + /// `POST v1/validator/aggregate_and_proofs` + pub async fn post_validator_aggregate_and_proof_v1( &self, aggregates: &[SignedAggregateAndProof], ) -> Result<(), Error> { @@ -2353,6 +2479,30 @@ impl BeaconNodeHttpClient { Ok(()) } + /// `POST v2/validator/aggregate_and_proofs` + pub async fn post_validator_aggregate_and_proof_v2( + &self, + aggregates: &[SignedAggregateAndProof], + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("validator") + .push("aggregate_and_proofs"); + + self.post_with_timeout_and_consensus_header( + path, + &aggregates, + self.timeouts.attestation, + fork_name, + ) + .await?; + + Ok(()) + } + /// `POST validator/beacon_committee_subscriptions` pub async fn post_validator_beacon_committee_subscriptions( &self, diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 70aa5aab3e2..d399bc2bd01 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -780,6 +780,8 @@ pub struct ValidatorAttestationDataQuery { pub struct ValidatorAggregateAttestationQuery { pub attestation_data_root: Hash256, pub slot: Slot, + #[serde(skip_serializing_if = "Option::is_none")] + pub committee_index: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 88993267a94..7b53a98caa1 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -1,6 +1,6 @@ use crate::slot_data::SlotData; -use crate::Checkpoint; use crate::{test_utils::TestRandom, Hash256, Slot}; +use crate::{Checkpoint, ForkVersionDeserialize}; use derivative::Derivative; use safe_arith::ArithError; use serde::{Deserialize, Serialize}; @@ -26,6 +26,12 @@ pub enum Error { InvalidCommitteeIndex, } +impl From for Error { + fn from(e: ssz_types::Error) -> Self { + Error::SszTypesError(e) + } +} + #[superstruct( variants(Base, Electra), variant_attributes( @@ -487,6 +493,46 @@ impl<'a, E: EthSpec> From> for AttestationRef<'a, E> } } +impl ForkVersionDeserialize for Attestation { + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::Value, + fork_name: crate::ForkName, + ) -> Result { + if fork_name.electra_enabled() { + let attestation: AttestationElectra = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(Attestation::Electra(attestation)) + } else { + let attestation: AttestationBase = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(Attestation::Base(attestation)) + } + } +} + +impl ForkVersionDeserialize for Vec> { + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::Value, + fork_name: crate::ForkName, + ) -> Result { + if fork_name.electra_enabled() { + let attestations: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(attestations + .into_iter() + .map(Attestation::Electra) + .collect::>()) + } else { + let attestations: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(attestations + .into_iter() + .map(Attestation::Base) + .collect::>()) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/attester_slashing.rs b/consensus/types/src/attester_slashing.rs index c8e2fb4f821..f6aa654d445 100644 --- a/consensus/types/src/attester_slashing.rs +++ b/consensus/types/src/attester_slashing.rs @@ -171,6 +171,29 @@ impl TestRandom for AttesterSlashing { } } +impl crate::ForkVersionDeserialize for Vec> { + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::Value, + fork_name: crate::ForkName, + ) -> Result { + if fork_name.electra_enabled() { + let slashings: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(slashings + .into_iter() + .map(AttesterSlashing::Electra) + .collect::>()) + } else { + let slashings: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(slashings + .into_iter() + .map(AttesterSlashing::Base) + .collect::>()) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 0cff39546dd..30fe508a2c2 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -287,17 +287,21 @@ impl AttestationService { // Then download, sign and publish a `SignedAggregateAndProof` for each // validator that is elected to aggregate for this `slot` and // `committee_index`. - self.produce_and_publish_aggregates(&attestation_data, &validator_duties) - .await - .map_err(move |e| { - crit!( - log, - "Error during attestation routine"; - "error" => format!("{:?}", e), - "committee_index" => committee_index, - "slot" => slot.as_u64(), - ) - })?; + self.produce_and_publish_aggregates( + &attestation_data, + committee_index, + &validator_duties, + ) + .await + .map_err(move |e| { + crit!( + log, + "Error during attestation routine"; + "error" => format!("{:?}", e), + "committee_index" => committee_index, + "slot" => slot.as_u64(), + ) + })?; } Ok(()) @@ -445,6 +449,11 @@ impl AttestationService { warn!(log, "No attestations were published"); return Ok(None); } + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); // Post the attestations to the BN. match self @@ -458,9 +467,15 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::ATTESTATIONS_HTTP_POST], ); - beacon_node - .post_beacon_pool_attestations(attestations) - .await + if fork_name.electra_enabled() { + beacon_node + .post_beacon_pool_attestations_v2(attestations, fork_name) + .await + } else { + beacon_node + .post_beacon_pool_attestations_v1(attestations) + .await + } }, ) .await @@ -504,6 +519,7 @@ impl AttestationService { async fn produce_and_publish_aggregates( &self, attestation_data: &AttestationData, + committee_index: CommitteeIndex, validator_duties: &[DutyAndProof], ) -> Result<(), String> { let log = self.context.log(); @@ -516,6 +532,12 @@ impl AttestationService { return Ok(()); } + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); + let aggregated_attestation = &self .beacon_nodes .first_success( @@ -526,17 +548,36 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::AGGREGATES_HTTP_GET], ); - beacon_node - .get_validator_aggregate_attestation( - attestation_data.slot, - attestation_data.tree_hash_root(), - ) - .await - .map_err(|e| { - format!("Failed to produce an aggregate attestation: {:?}", e) - })? - .ok_or_else(|| format!("No aggregate available for {:?}", attestation_data)) - .map(|result| result.data) + if fork_name.electra_enabled() { + beacon_node + .get_validator_aggregate_attestation_v2( + attestation_data.slot, + attestation_data.tree_hash_root(), + committee_index, + ) + .await + .map_err(|e| { + format!("Failed to produce an aggregate attestation: {:?}", e) + })? + .ok_or_else(|| { + format!("No aggregate available for {:?}", attestation_data) + }) + .map(|result| result.data) + } else { + beacon_node + .get_validator_aggregate_attestation_v1( + attestation_data.slot, + attestation_data.tree_hash_root(), + ) + .await + .map_err(|e| { + format!("Failed to produce an aggregate attestation: {:?}", e) + })? + .ok_or_else(|| { + format!("No aggregate available for {:?}", attestation_data) + }) + .map(|result| result.data) + } }, ) .await @@ -604,9 +645,20 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::AGGREGATES_HTTP_POST], ); - beacon_node - .post_validator_aggregate_and_proof(signed_aggregate_and_proofs_slice) - .await + if fork_name.electra_enabled() { + beacon_node + .post_validator_aggregate_and_proof_v2( + signed_aggregate_and_proofs_slice, + fork_name, + ) + .await + } else { + beacon_node + .post_validator_aggregate_and_proof_v1( + signed_aggregate_and_proofs_slice, + ) + .await + } }, ) .await From d88302a914920fa1cb20baa9c9b4e61ae0af220c Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 15 Jul 2024 15:03:50 -0700 Subject: [PATCH 214/227] fix ef test compile --- testing/ef_tests/src/cases/operations.rs | 8 ++++---- testing/ef_tests/src/type_name.rs | 2 +- testing/ef_tests/tests/tests.rs | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index bb82674c88a..09c1de941af 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -7,7 +7,7 @@ use ssz::Decode; use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache; use state_processing::epoch_cache::initialize_epoch_cache; use state_processing::per_block_processing::process_operations::{ - process_consolidations, process_deposit_receipts, process_execution_layer_withdrawal_requests, + process_consolidations, process_deposit_requests, process_execution_layer_withdrawal_requests, }; use state_processing::{ per_block_processing::{ @@ -25,7 +25,7 @@ use std::fmt::Debug; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockBodyBellatrix, BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconState, - BlindedPayload, Deposit, DepositReceipt, ExecutionLayerWithdrawalRequest, ExecutionPayload, + BlindedPayload, Deposit, DepositRequest, ExecutionLayerWithdrawalRequest, ExecutionPayload, FullPayload, ProposerSlashing, SignedBlsToExecutionChange, SignedConsolidation, SignedVoluntaryExit, SyncAggregate, }; @@ -468,7 +468,7 @@ impl Operation for ExecutionLayerWithdrawalRequest { } } -impl Operation for DepositReceipt { +impl Operation for DepositRequest { fn handler_name() -> String { "deposit_receipt".into() } @@ -487,7 +487,7 @@ impl Operation for DepositReceipt { spec: &ChainSpec, _extra: &Operations, ) -> Result<(), BlockProcessingError> { - process_deposit_receipts(state, &[self.clone()], spec) + process_deposit_requests(state, &[self.clone()], spec) } } diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index dd12dacafa2..d6ef873ead4 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -65,7 +65,7 @@ type_name_generic!(ContributionAndProof); type_name!(Deposit); type_name!(DepositData); type_name!(DepositMessage); -type_name!(DepositReceipt); +type_name!(DepositRequest); type_name!(Eth1Data); type_name!(ExecutionLayerWithdrawalRequest); type_name_generic!(ExecutionPayload); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 7a7c264682e..96631d840c8 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -101,8 +101,8 @@ fn operations_execution_layer_withdrawal_reqeusts() { #[test] #[cfg(not(feature = "fake_crypto"))] fn operations_deposit_receipts() { - OperationsHandler::::default().run(); - OperationsHandler::::default().run(); + OperationsHandler::::default().run(); + OperationsHandler::::default().run(); } #[test] @@ -242,7 +242,7 @@ mod ssz_static { use types::blob_sidecar::BlobIdentifier; use types::historical_summary::HistoricalSummary; use types::{ - AttesterSlashingBase, AttesterSlashingElectra, Consolidation, DepositReceipt, + AttesterSlashingBase, AttesterSlashingElectra, Consolidation, DepositRequest, ExecutionLayerWithdrawalRequest, LightClientBootstrapAltair, PendingBalanceDeposit, PendingPartialWithdrawal, *, }; @@ -636,8 +636,8 @@ mod ssz_static { #[test] fn deposit_receipt() { - SszStaticHandler::::electra_and_later().run(); - SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); } #[test] From c48cd52a9d4a529cb39ee62d4629aca617226f13 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 15 Jul 2024 15:08:57 -0700 Subject: [PATCH 215/227] =?UTF-8?q?use=20`enabled`=C2=A0fork=20methods?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- testing/ef_tests/src/cases/operations.rs | 14 +++++++------- testing/ef_tests/src/handler.rs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 09c1de941af..551f8281428 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -298,7 +298,7 @@ impl Operation for BeaconBlockBody> { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name >= ForkName::Bellatrix + fork_name.bellatrix_enabled() } fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -340,7 +340,7 @@ impl Operation for BeaconBlockBody> { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name >= ForkName::Bellatrix + fork_name.bellatrix_enabled() } fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -396,7 +396,7 @@ impl Operation for WithdrawalsPayload { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name >= ForkName::Capella + fork_name.capella_enabled() } fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -428,7 +428,7 @@ impl Operation for SignedBlsToExecutionChange { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name >= ForkName::Capella + fork_name.capella_enabled() } fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -451,7 +451,7 @@ impl Operation for ExecutionLayerWithdrawalRequest { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name >= ForkName::Electra + fork_name.electra_enabled() } fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -474,7 +474,7 @@ impl Operation for DepositRequest { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name >= ForkName::Electra + fork_name.electra_enabled() } fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -497,7 +497,7 @@ impl Operation for SignedConsolidation { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name >= ForkName::Electra + fork_name.electra_enabled() } fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index fa895b1765c..27ddbc0203a 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -620,7 +620,7 @@ impl Handler for OptimisticSyncHandler { } fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool { - fork_name >= ForkName::Bellatrix && cfg!(not(feature = "fake_crypto")) + fork_name.bellatrix_enabled() && cfg!(not(feature = "fake_crypto")) } } From c0fd66f5e96cc39ca6c0426c4df321364778b54e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 15 Jul 2024 15:37:34 -0700 Subject: [PATCH 216/227] get tests passing --- beacon_node/beacon_chain/src/eth1_chain.rs | 4 +- consensus/types/src/beacon_state.rs | 6 ++- consensus/types/src/deposit_receipt.rs | 37 ------------------- consensus/types/src/execution_payload.rs | 2 + .../types/src/execution_payload_header.rs | 8 ++-- consensus/types/src/lib.rs | 4 +- testing/ef_tests/check_all_files_accessed.py | 7 +++- testing/ef_tests/src/cases/operations.rs | 2 +- testing/ef_tests/tests/tests.rs | 20 +++++----- 9 files changed, 33 insertions(+), 57 deletions(-) delete mode 100644 consensus/types/src/deposit_receipt.rs diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index 62aad558eeb..b4005f22fd1 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -548,8 +548,8 @@ impl Eth1ChainBackend for CachingEth1Backend { // [New in Electra:EIP6110] let deposit_index_limit = - if let Ok(deposit_receipts_start_index) = state.deposit_requests_start_index() { - std::cmp::min(deposit_count, deposit_receipts_start_index) + if let Ok(deposit_requests_start_index) = state.deposit_requests_start_index() { + std::cmp::min(deposit_count, deposit_requests_start_index) } else { deposit_count }; diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 74241599723..054e5dbe271 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -486,7 +486,11 @@ where // Electra #[superstruct(only(Electra), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] - #[serde(with = "serde_utils::quoted_u64")] + #[serde( + with = "serde_utils::quoted_u64", + //TODO(electra) remove alias when ef tests are updated + alias = "deposit_receipts_start_index" + )] pub deposit_requests_start_index: u64, #[superstruct(only(Electra), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] diff --git a/consensus/types/src/deposit_receipt.rs b/consensus/types/src/deposit_receipt.rs deleted file mode 100644 index f6ddf8b63a8..00000000000 --- a/consensus/types/src/deposit_receipt.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::test_utils::TestRandom; -use crate::{Hash256, PublicKeyBytes, Signature}; -use serde::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode}; -use test_random_derive::TestRandom; -use tree_hash_derive::TreeHash; - -#[derive( - arbitrary::Arbitrary, - Debug, - PartialEq, - Eq, - Hash, - Clone, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, -)] -pub struct DepositRequest { - pub pubkey: PublicKeyBytes, - pub withdrawal_credentials: Hash256, - #[serde(with = "serde_utils::quoted_u64")] - pub amount: u64, - pub signature: Signature, - #[serde(with = "serde_utils::quoted_u64")] - pub index: u64, -} - -#[cfg(test)] -mod tests { - use super::*; - - ssz_and_tree_hash_tests!(DepositRequest); -} diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index eca561f7353..02300cc1927 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -94,6 +94,8 @@ pub struct ExecutionPayload { #[serde(with = "serde_utils::quoted_u64")] pub excess_blob_gas: u64, #[superstruct(only(Electra))] + //TODO(electra) remove alias once EF tests are updates with correct name + #[serde(alias = "deposit_receipts")] pub deposit_requests: VariableList, #[superstruct(only(Electra))] pub withdrawal_requests: diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 962e7a16fbc..149cc286ae9 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -86,7 +86,9 @@ pub struct ExecutionPayloadHeader { #[serde(with = "serde_utils::quoted_u64")] pub excess_blob_gas: u64, #[superstruct(only(Electra), partial_getter(copy))] - pub deposit_receipts_root: Hash256, + //TODO(electra) remove alias once EF tests are updates with correct name + #[serde(alias = "deposit_receipts_root")] + pub deposit_requests_root: Hash256, #[superstruct(only(Electra), partial_getter(copy))] pub withdrawal_requests_root: Hash256, } @@ -202,7 +204,7 @@ impl ExecutionPayloadHeaderDeneb { withdrawals_root: self.withdrawals_root, blob_gas_used: self.blob_gas_used, excess_blob_gas: self.excess_blob_gas, - deposit_receipts_root: Hash256::zero(), + deposit_requests_root: Hash256::zero(), withdrawal_requests_root: Hash256::zero(), } } @@ -295,7 +297,7 @@ impl<'a, E: EthSpec> From<&'a ExecutionPayloadElectra> for ExecutionPayloadHe withdrawals_root: payload.withdrawals.tree_hash_root(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, - deposit_receipts_root: payload.deposit_requests.tree_hash_root(), + deposit_requests_root: payload.deposit_requests.tree_hash_root(), withdrawal_requests_root: payload.withdrawal_requests.tree_hash_root(), } } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index c8c37ad708e..b5c500f0b22 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -33,7 +33,7 @@ pub mod contribution_and_proof; pub mod deposit; pub mod deposit_data; pub mod deposit_message; -pub mod deposit_receipt; +pub mod deposit_request; pub mod deposit_tree_snapshot; pub mod enr_fork_id; pub mod eth1_data; @@ -150,7 +150,7 @@ pub use crate::contribution_and_proof::ContributionAndProof; pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH}; pub use crate::deposit_data::DepositData; pub use crate::deposit_message::DepositMessage; -pub use crate::deposit_receipt::DepositRequest; +pub use crate::deposit_request::DepositRequest; pub use crate::deposit_tree_snapshot::{DepositTreeSnapshot, FinalizedExecutionBlock}; pub use crate::enr_fork_id::EnrForkId; pub use crate::epoch_cache::{EpochCache, EpochCacheError, EpochCacheKey}; diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index d0a19699d58..e1a308f7a40 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -44,10 +44,13 @@ "tests/.*/eip6110", "tests/.*/whisk", "tests/.*/eip7594", - # re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved + # TODO(electra) re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved "tests/.*/electra/ssz_static/LightClientUpdate", "tests/.*/electra/ssz_static/LightClientFinalityUpdate", - "tests/.*/electra/ssz_static/LightClientBootstrap" + "tests/.*/electra/ssz_static/LightClientBootstrap", + # TODO(electra) re-enable as DepositRequest when EF tests are updated + "tests/.*/electra/operations/deposit_receipt", + "tests/.*/electra/ssz_static/DepositReceipt" ] diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 551f8281428..0af2c818271 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -470,7 +470,7 @@ impl Operation for ExecutionLayerWithdrawalRequest { impl Operation for DepositRequest { fn handler_name() -> String { - "deposit_receipt".into() + "deposit_request".into() } fn is_enabled_for_fork(fork_name: ForkName) -> bool { diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 96631d840c8..1809b01dc21 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -100,9 +100,10 @@ fn operations_execution_layer_withdrawal_reqeusts() { #[test] #[cfg(not(feature = "fake_crypto"))] -fn operations_deposit_receipts() { - OperationsHandler::::default().run(); - OperationsHandler::::default().run(); +fn operations_deposit_requests() { + //TODO(electra): re-enable mainnet once they update the name for this + // OperationsHandler::::default().run(); + // OperationsHandler::::default().run(); } #[test] @@ -242,7 +243,7 @@ mod ssz_static { use types::blob_sidecar::BlobIdentifier; use types::historical_summary::HistoricalSummary; use types::{ - AttesterSlashingBase, AttesterSlashingElectra, Consolidation, DepositRequest, + AttesterSlashingBase, AttesterSlashingElectra, Consolidation, ExecutionLayerWithdrawalRequest, LightClientBootstrapAltair, PendingBalanceDeposit, PendingPartialWithdrawal, *, }; @@ -634,11 +635,12 @@ mod ssz_static { SszStaticHandler::::electra_and_later().run(); } - #[test] - fn deposit_receipt() { - SszStaticHandler::::electra_and_later().run(); - SszStaticHandler::::electra_and_later().run(); - } + // TODO(electra) re-enable when EF tests are updated + // #[test] + // fn deposit_request() { + // SszStaticHandler::::electra_and_later().run(); + // SszStaticHandler::::electra_and_later().run(); + // } #[test] fn execution_layer_withdrawal_request() { From 064c4611b944af04d33c904b54a936d7defe3270 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 15 Jul 2024 15:41:59 -0700 Subject: [PATCH 217/227] last updates --- testing/ef_tests/src/handler.rs | 2 +- testing/ef_tests/tests/tests.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index 27ddbc0203a..410a37e7682 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -584,7 +584,7 @@ impl Handler for ForkChoiceHandler { // No FCU override tests prior to bellatrix. if self.handler_name == "should_override_forkchoice_update" - && fork_name < ForkName::Bellatrix + && !fork_name.bellatrix_enabled() { return false; } diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 1809b01dc21..10a57a6b45e 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -397,7 +397,7 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); - // Re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved + // TODO(electra) re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved // SszStaticHandler::, MinimalEthSpec>::electra_only() // .run(); // SszStaticHandler::, MainnetEthSpec>::electra_only() @@ -477,7 +477,7 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::deneb_only( ) .run(); - // Re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved + // TODO(electra) re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved // SszStaticHandler::, MinimalEthSpec>::electra_only( // ) // .run(); @@ -507,7 +507,7 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); - // Re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved + // TODO(electra) re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved // SszStaticHandler::, MinimalEthSpec>::electra_only( // ) // .run(); From 617a349957ecb5695ec99e191eff04897228212a Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Mon, 15 Jul 2024 18:42:13 -0500 Subject: [PATCH 218/227] Fix bug on engine api with consolidations (#6101) --- beacon_node/execution_layer/src/engine_api.rs | 11 +++-- .../src/engine_api/json_structures.rs | 44 +++++++++++++++++-- beacon_node/execution_layer/src/lib.rs | 11 +++-- .../src/test_utils/handle_rpc.rs | 8 +++- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index d5c00e4dbe6..5322355ddae 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -43,7 +43,7 @@ pub use new_payload_request::{ NewPayloadRequestDeneb, NewPayloadRequestElectra, }; -use self::json_structures::{JsonDepositRequest, JsonWithdrawalRequest}; +use self::json_structures::{JsonConsolidationRequest, JsonDepositRequest, JsonWithdrawalRequest}; pub const LATEST_TAG: &str = "latest"; @@ -210,8 +210,7 @@ pub struct ExecutionBlockWithTransactions { #[superstruct(only(Electra))] pub withdrawal_requests: Vec, #[superstruct(only(Electra))] - // TODO(electra): I don't think we need a JsonConsolidationRequest here because the bytes should be little-endian but we need to confirm - pub consolidation_requests: Vec, + pub consolidation_requests: Vec, } impl TryFrom> for ExecutionBlockWithTransactions { @@ -329,7 +328,11 @@ impl TryFrom> for ExecutionBlockWithTransactions .into_iter() .map(|withdrawal| withdrawal.into()) .collect(), - consolidation_requests: block.consolidation_requests.to_vec(), + consolidation_requests: block + .consolidation_requests + .into_iter() + .map(Into::into) + .collect(), }) } }; diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index bfac5c87dd9..0eb20484141 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -108,7 +108,7 @@ pub struct JsonExecutionPayload { VariableList, #[superstruct(only(V4))] pub consolidation_requests: - VariableList, + VariableList, } impl From> for JsonExecutionPayloadV1 { @@ -223,7 +223,12 @@ impl From> for JsonExecutionPayloadV4 .map(Into::into) .collect::>() .into(), - consolidation_requests: payload.consolidation_requests, + consolidation_requests: payload + .consolidation_requests + .into_iter() + .map(Into::into) + .collect::>() + .into(), } } } @@ -352,7 +357,12 @@ impl From> for ExecutionPayloadElectra .map(Into::into) .collect::>() .into(), - consolidation_requests: payload.consolidation_requests, + consolidation_requests: payload + .consolidation_requests + .into_iter() + .map(Into::into) + .collect::>() + .into(), } } } @@ -927,3 +937,31 @@ impl From for WithdrawalRequest { } } } + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct JsonConsolidationRequest { + pub source_address: Address, + pub source_pubkey: PublicKeyBytes, + pub target_pubkey: PublicKeyBytes, +} + +impl From for JsonConsolidationRequest { + fn from(consolidation_request: ConsolidationRequest) -> Self { + Self { + source_address: consolidation_request.source_address, + source_pubkey: consolidation_request.source_pubkey, + target_pubkey: consolidation_request.target_pubkey, + } + } +} + +impl From for ConsolidationRequest { + fn from(json_consolidation_request: JsonConsolidationRequest) -> Self { + Self { + source_address: json_consolidation_request.source_address, + source_pubkey: json_consolidation_request.source_pubkey, + target_pubkey: json_consolidation_request.target_pubkey, + } + } +} diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index e40e948cc6b..371aa94e19b 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -2011,9 +2011,14 @@ impl ExecutionLayer { ) .map_err(ApiError::DeserializeWithdrawalRequests)?; let n_consolidations = electra_block.consolidation_requests.len(); - let consolidation_requests = - VariableList::new(electra_block.consolidation_requests) - .map_err(|_| ApiError::TooManyConsolidationRequests(n_consolidations))?; + let consolidation_requests = VariableList::new( + electra_block + .consolidation_requests + .into_iter() + .map(Into::into) + .collect::>(), + ) + .map_err(|_| ApiError::TooManyConsolidationRequests(n_consolidations))?; ExecutionPayload::Electra(ExecutionPayloadElectra { parent_hash: electra_block.parent_hash, fee_recipient: electra_block.fee_recipient, diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index d86b4439ad7..b1788498424 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -658,7 +658,13 @@ pub async fn handle_rpc( ), consolidation_requests: block.consolidation_requests().ok().map( |consolidation_requests| { - VariableList::from(consolidation_requests.clone()) + VariableList::from( + consolidation_requests + .clone() + .into_iter() + .map(Into::into) + .collect::>(), + ) }, ), })); From 8bff9669b277fe0c562fe73d05ea98e2a380fa11 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Tue, 16 Jul 2024 11:10:56 -0500 Subject: [PATCH 219/227] Fix small bug in engine_api --- beacon_node/execution_layer/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 371aa94e19b..194254cdbf2 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1791,7 +1791,7 @@ impl ExecutionLayer { .request(|engine: &Engine| async move { engine .api - .get_payload_bodies_by_range_v1(start, count) + .get_payload_bodies_by_range(start, count) .await }) .await From 42c8c45de69239a835e04c8f4c03f042d3fe98ef Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 16 Jul 2024 09:45:06 -0700 Subject: [PATCH 220/227] add deposit request file --- consensus/types/src/deposit_request.rs | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 consensus/types/src/deposit_request.rs diff --git a/consensus/types/src/deposit_request.rs b/consensus/types/src/deposit_request.rs new file mode 100644 index 00000000000..f6ddf8b63a8 --- /dev/null +++ b/consensus/types/src/deposit_request.rs @@ -0,0 +1,37 @@ +use crate::test_utils::TestRandom; +use crate::{Hash256, PublicKeyBytes, Signature}; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + arbitrary::Arbitrary, + Debug, + PartialEq, + Eq, + Hash, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, +)] +pub struct DepositRequest { + pub pubkey: PublicKeyBytes, + pub withdrawal_credentials: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub amount: u64, + pub signature: Signature, + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_and_tree_hash_tests!(DepositRequest); +} From 6127b4bfa0ff30d48b6757818efd9554bc07cbb6 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 16 Jul 2024 16:07:58 -0700 Subject: [PATCH 221/227] clean up after merge --- beacon_node/execution_layer/src/lib.rs | 5 +---- testing/ef_tests/check_all_files_accessed.py | 5 +---- testing/ef_tests/tests/tests.rs | 4 ++-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 194254cdbf2..0da9225173c 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1789,10 +1789,7 @@ impl ExecutionLayer { let _timer = metrics::start_timer(&metrics::EXECUTION_LAYER_GET_PAYLOAD_BODIES_BY_RANGE); self.engine() .request(|engine: &Engine| async move { - engine - .api - .get_payload_bodies_by_range(start, count) - .await + engine.api.get_payload_bodies_by_range(start, count).await }) .await .map_err(Box::new) diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index e1a308f7a40..58b845b7664 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -47,10 +47,7 @@ # TODO(electra) re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved "tests/.*/electra/ssz_static/LightClientUpdate", "tests/.*/electra/ssz_static/LightClientFinalityUpdate", - "tests/.*/electra/ssz_static/LightClientBootstrap", - # TODO(electra) re-enable as DepositRequest when EF tests are updated - "tests/.*/electra/operations/deposit_receipt", - "tests/.*/electra/ssz_static/DepositReceipt" + "tests/.*/electra/ssz_static/LightClientBootstrap" ] diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 9f551645e41..7b227fffaf2 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -629,8 +629,8 @@ mod ssz_static { #[test] fn consolidation() { - SszStaticHandler::::electra_and_later().run(); - SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); + SszStaticHandler::::electra_and_later().run(); } #[test] From d1b2726b9cca077f915f96ec399ad2174d6b4989 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Thu, 18 Jul 2024 11:29:51 -0500 Subject: [PATCH 222/227] Fix Broken Test on `devnet-1` --- consensus/fork_choice/tests/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index d2935dbca45..19faf69bcb4 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -1344,7 +1344,7 @@ async fn progressive_balances_cache_attester_slashing() { // (`HeaderInvalid::ProposerSlashed`). The harness should be re-worked to successfully skip // the slot in this scenario rather than panic-ing. The same applies to // `progressive_balances_cache_proposer_slashing`. - .apply_blocks(1) + .apply_blocks(2) .await .add_previous_epoch_attester_slashing() .await From 9e8b71e4c3d9ee4f8d9864d26ae5ee3f99cd5647 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Thu, 18 Jul 2024 14:56:32 -0500 Subject: [PATCH 223/227] another small bug --- beacon_node/execution_layer/src/engine_api/http.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 73eabf574b5..dd04a9b35b2 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -1111,7 +1111,7 @@ impl HttpJsonRpc { struct Quantity(#[serde(with = "serde_utils::u64_hex_be")] u64); let params = json!([Quantity(start), Quantity(count)]); - let response: Vec>> = self + let response: Vec>> = self .rpc_request( ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V2, params, @@ -1121,7 +1121,7 @@ impl HttpJsonRpc { Ok(response .into_iter() - .map(|opt_json| opt_json.map(|v2| JsonExecutionPayloadBody::V1(v2).into())) + .map(|opt_json| opt_json.map(|v2| JsonExecutionPayloadBody::V2(v2).into())) .collect()) } From 63e418aa013c79bed3229c706ea0b40084e985e4 Mon Sep 17 00:00:00 2001 From: Mark Mackey Date: Tue, 6 Aug 2024 11:01:08 -0500 Subject: [PATCH 224/227] Fix Deserialization Bug in `engine-api` --- .../execution_layer/src/engine_api/json_structures.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 0eb20484141..665ca38d46e 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -731,13 +731,14 @@ impl From for JsonForkchoiceUpdatedV1Response { #[superstruct( variants(V1, V2), - variant_attributes(derive(Clone, Debug, Serialize, Deserialize),), + variant_attributes( + derive(Clone, Debug, Serialize, Deserialize), + serde(bound = "E: EthSpec", rename_all = "camelCase"), + ), partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant") )] #[derive(Clone, Debug, Serialize)] -#[serde(bound = "E: EthSpec")] -#[serde(rename_all = "camelCase")] -#[serde(untagged)] +#[serde(bound = "E: EthSpec", rename_all = "camelCase", untagged)] pub struct JsonExecutionPayloadBody { #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, From 51d51f5dcd019643660a3cc2411aeec744fa1e60 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 6 Aug 2024 09:14:38 -0700 Subject: [PATCH 225/227] remove import --- consensus/types/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index f854929a93a..2b874be4825 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -162,7 +162,6 @@ pub use crate::eth1_data::Eth1Data; pub use crate::eth_spec::EthSpecId; pub use crate::execution_block_hash::ExecutionBlockHash; pub use crate::execution_block_header::{EncodableExecutionBlockHeader, ExecutionBlockHeader}; -pub use crate::execution_layer_withdrawal_request::ExecutionLayerWithdrawalRequest; pub use crate::execution_payload::{ ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadRef, Transaction, Transactions, Withdrawals, From b574838fa762233f8d0691eb6dad944d8c3dbe75 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 16 Aug 2024 15:57:14 +1000 Subject: [PATCH 226/227] Remove false negative checks on `shuffling_is_compatible` --- beacon_node/beacon_chain/tests/store_tests.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 7049bf14fde..c1071d55cf6 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -1437,18 +1437,16 @@ fn check_shuffling_compatible( head_state.current_epoch(), |committee_cache, _| { let state_cache = head_state.committee_cache(RelativeEpoch::Current).unwrap(); + // We used to check for false negatives here, but had to remove that check + // because `shuffling_is_compatible` does not guarantee their absence. + // + // See: https://github.com/sigp/lighthouse/issues/6269 if current_epoch_shuffling_is_compatible { assert_eq!( committee_cache, state_cache.as_ref(), "block at slot {slot}" ); - } else { - assert_ne!( - committee_cache, - state_cache.as_ref(), - "block at slot {slot}" - ); } Ok(()) }, @@ -1479,8 +1477,6 @@ fn check_shuffling_compatible( let state_cache = head_state.committee_cache(RelativeEpoch::Previous).unwrap(); if previous_epoch_shuffling_is_compatible { assert_eq!(committee_cache, state_cache.as_ref()); - } else { - assert_ne!(committee_cache, state_cache.as_ref()); } Ok(()) }, From 1e95ab616acccd6af4f8a9190b4e15e0557015da Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 22 Aug 2024 13:15:23 +1000 Subject: [PATCH 227/227] Fix flaky validator monitor test --- .../beacon_chain/tests/validator_monitor.rs | 42 ++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/beacon_node/beacon_chain/tests/validator_monitor.rs b/beacon_node/beacon_chain/tests/validator_monitor.rs index c2c03baff04..b4a54d26676 100644 --- a/beacon_node/beacon_chain/tests/validator_monitor.rs +++ b/beacon_node/beacon_chain/tests/validator_monitor.rs @@ -4,7 +4,7 @@ use beacon_chain::test_utils::{ use beacon_chain::validator_monitor::{ValidatorMonitorConfig, MISSED_BLOCK_LAG_SLOTS}; use logging::test_logger; use std::sync::LazyLock; -use types::{Epoch, EthSpec, Keypair, MainnetEthSpec, PublicKeyBytes, Slot}; +use types::{Epoch, EthSpec, ForkName, Keypair, MainnetEthSpec, PublicKeyBytes, Slot}; // Should ideally be divisible by 3. pub const VALIDATOR_COUNT: usize = 48; @@ -128,7 +128,7 @@ async fn produces_missed_blocks() { let initial_blocks = slots_per_epoch * nb_epoch_to_simulate.as_u64() - 1; // The validator index of the validator that is 'supposed' to miss a block - let mut validator_index_to_monitor = 1; + let validator_index_to_monitor = 1; // 1st scenario // // @@ -201,34 +201,22 @@ async fn produces_missed_blocks() { // Missed block happens when slot and prev_slot are not in the same epoch // making sure that the cache reloads when the epoch changes // in that scenario the slot that missed a block is the first slot of the epoch - validator_index_to_monitor = 7; // We are adding other validators to monitor as these ones will miss a block depending on // the fork name specified when running the test as the proposer cache differs depending on // the fork name (cf. seed) // // If you are adding a new fork and seeing errors, print // `validator_indexes[slot_in_epoch.as_usize()]` and add it below. - let validator_index_to_monitor_altair = 2; - // Same as above but for the merge upgrade - let validator_index_to_monitor_bellatrix = 4; - // Same as above but for the capella upgrade - let validator_index_to_monitor_capella = 11; - // Same as above but for the deneb upgrade - let validator_index_to_monitor_deneb = 3; - // Same as above but for the electra upgrade - let validator_index_to_monitor_electra = 6; - - let harness2 = get_harness( - validator_count, - vec![ - validator_index_to_monitor, - validator_index_to_monitor_altair, - validator_index_to_monitor_bellatrix, - validator_index_to_monitor_capella, - validator_index_to_monitor_deneb, - validator_index_to_monitor_electra, - ], - ); + let validator_index_to_monitor = match harness1.spec.fork_name_at_slot::(Slot::new(0)) { + ForkName::Base => 7, + ForkName::Altair => 2, + ForkName::Bellatrix => 4, + ForkName::Capella => 11, + ForkName::Deneb => 3, + ForkName::Electra => 1, + }; + + let harness2 = get_harness(validator_count, vec![validator_index_to_monitor]); let advance_slot_by = 9; harness2 .extend_chain( @@ -300,6 +288,12 @@ async fn produces_missed_blocks() { duplicate_block_root = *_state2.block_roots().get(idx as usize).unwrap(); validator_indexes = _state2.get_beacon_proposer_indices(&harness2.spec).unwrap(); let not_monitored_validator_index = validator_indexes[slot_in_epoch.as_usize()]; + // This could do with a refactor: https://github.com/sigp/lighthouse/issues/6293 + assert_ne!( + not_monitored_validator_index, + validator_index_to_monitor, + "this test has a fragile dependency on hardcoded indices. you need to tweak some settings or rewrite this" + ); assert_eq!( _state2.set_block_root(prev_slot, duplicate_block_root),