[stateless validation] Real ChunkStateWitness struct, and prototype v…

…alidation logic (#10381)

This fills in the ChunkStateWitness struct with real data that would be
included in a chunk state witness.

It also includes a prototype implementation for validating the
ChunkStateWitness against the chain, which includes a pre-validation
part where the ChainStore is used to retrieve necessary blocks and chunk
headers (to verify receipts and transactions against), as well as the
main validation part where transactions and receipts are applied with
the runtime.

This verification logic is untested right now because the state witness
production part is not implemented.

#9292

chain/chain-primitives/src/error.rs

@@ -135,7 +135,7 @@ pub enum Error {
     #[error("Invalid Chunk State")]
     InvalidChunkState(Box<ChunkState>),
     #[error("Invalid Chunk State Witness")]
-    InvalidChunkStateWitness,
+    InvalidChunkStateWitness(String),
     /// Invalid chunk mask
     #[error("Invalid Chunk Mask")]
     InvalidChunkMask,
@@ -270,7 +270,7 @@ impl Error {
             | Error::InvalidChunk
             | Error::InvalidChunkProofs(_)
             | Error::InvalidChunkState(_)
-            | Error::InvalidChunkStateWitness
+            | Error::InvalidChunkStateWitness(_)
             | Error::InvalidChunkMask
             | Error::InvalidStateRoot
             | Error::InvalidTxRoot
@@ -343,7 +343,7 @@ impl Error {
             Error::InvalidChunk => "invalid_chunk",
             Error::InvalidChunkProofs(_) => "invalid_chunk_proofs",
             Error::InvalidChunkState(_) => "invalid_chunk_state",
-            Error::InvalidChunkStateWitness => "invalid_chunk_state_witness",
+            Error::InvalidChunkStateWitness(_) => "invalid_chunk_state_witness",
             Error::InvalidChunkMask => "invalid_chunk_mask",
             Error::InvalidStateRoot => "invalid_state_root",
             Error::InvalidTxRoot => "invalid_tx_root",

chain/chain/src/chain.rs

@@ -8,6 +8,7 @@ use crate::lightclient::get_epoch_block_producers_view;
 use crate::migrations::check_if_block_is_first_with_chunk_of_version;
 use crate::missing_chunks::MissingChunksPool;
 use crate::orphan::{Orphan, OrphanBlockPool};
+use crate::sharding::shuffle_receipt_proofs;
 use crate::state_request_tracker::StateRequestTracker;
 use crate::state_snapshot_actor::SnapshotCallbacks;
 use crate::store::{ChainStore, ChainStoreAccess, ChainStoreUpdate};
@@ -93,9 +94,6 @@ use near_store::flat::{store_helper, FlatStorageReadyStatus, FlatStorageStatus};
 use near_store::get_genesis_state_roots;
 use near_store::DBCol;
 use once_cell::sync::OnceCell;
-use rand::seq::SliceRandom;
-use rand::SeedableRng;
-use rand_chacha::ChaCha20Rng;
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fmt::{Debug, Formatter};
@@ -1275,22 +1273,12 @@ impl Chain {
         }
         // sort the receipts deterministically so the order that they will be processed is deterministic
         for (_, receipt_proofs) in receipt_proofs_by_shard_id.iter_mut() {
-            Self::shuffle_receipt_proofs(receipt_proofs, block.hash());
+            shuffle_receipt_proofs(receipt_proofs, block.hash());
         }
 
         Ok(receipt_proofs_by_shard_id)
     }
 
-    fn shuffle_receipt_proofs<ReceiptProofType>(
-        receipt_proofs: &mut Vec<ReceiptProofType>,
-        block_hash: &CryptoHash,
-    ) {
-        let mut slice = [0u8; 32];
-        slice.copy_from_slice(block_hash.as_ref());
-        let mut rng: ChaCha20Rng = SeedableRng::from_seed(slice);
-        receipt_proofs.shuffle(&mut rng);
-    }
-
     /// Start processing a received or produced block. This function will process block asynchronously.
     /// It preprocesses the block by verifying that the block is valid and ready to process, then
     /// schedules the work of applying chunks in rayon thread pool. The function will return before
@@ -4714,19 +4702,3 @@ impl Chain {
             .collect()
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use near_primitives::hash::CryptoHash;
-
-    #[test]
-    pub fn receipt_randomness_reproducibility() {
-        // Sanity check that the receipt shuffling implementation does not change.
-        let mut receipt_proofs = vec![0, 1, 2, 3, 4, 5, 6];
-        crate::Chain::shuffle_receipt_proofs(
-            &mut receipt_proofs,
-            &CryptoHash::hash_bytes(&[1, 2, 3, 4, 5]),
-        );
-        assert_eq!(receipt_proofs, vec![2, 3, 1, 4, 0, 5, 6],);
-    }
-}

chain/chain/src/lib.rs

@@ -32,6 +32,7 @@ pub mod test_utils;
 pub mod types;
 pub mod validate;
 
+pub mod sharding;
 #[cfg(test)]
 mod tests;
 mod update_shard;

chain/chain/src/sharding.rs

@@ -0,0 +1,28 @@
+use near_primitives::hash::CryptoHash;
+use rand::seq::SliceRandom;
+use rand::SeedableRng;
+use rand_chacha::ChaCha20Rng;
+
+pub fn shuffle_receipt_proofs<ReceiptProofType>(
+    receipt_proofs: &mut Vec<ReceiptProofType>,
+    block_hash: &CryptoHash,
+) {
+    let mut slice = [0u8; 32];
+    slice.copy_from_slice(block_hash.as_ref());
+    let mut rng: ChaCha20Rng = SeedableRng::from_seed(slice);
+    receipt_proofs.shuffle(&mut rng);
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::sharding::shuffle_receipt_proofs;
+    use near_primitives::hash::CryptoHash;
+
+    #[test]
+    pub fn receipt_randomness_reproducibility() {
+        // Sanity check that the receipt shuffling implementation does not change.
+        let mut receipt_proofs = vec![0, 1, 2, 3, 4, 5, 6];
+        shuffle_receipt_proofs(&mut receipt_proofs, &CryptoHash::hash_bytes(&[1, 2, 3, 4, 5]));
+        assert_eq!(receipt_proofs, vec![2, 3, 1, 4, 0, 5, 6],);
+    }
+}

chain/chain/src/validate.rs

@@ -111,6 +111,31 @@ pub fn validate_chunk_with_chunk_extra(
     prev_chunk_extra: &ChunkExtra,
     prev_chunk_height_included: BlockHeight,
     chunk_header: &ShardChunkHeader,
+) -> Result<(), Error> {
+    let outgoing_receipts = chain_store.get_outgoing_receipts_for_shard(
+        epoch_manager,
+        *prev_block_hash,
+        chunk_header.shard_id(),
+        prev_chunk_height_included,
+    )?;
+    let outgoing_receipts_hashes = {
+        let shard_layout = epoch_manager.get_shard_layout_from_prev_block(prev_block_hash)?;
+        Chain::build_receipts_hashes(&outgoing_receipts, &shard_layout)
+    };
+    let (outgoing_receipts_root, _) = merklize(&outgoing_receipts_hashes);
+
+    validate_chunk_with_chunk_extra_and_receipts_root(
+        prev_chunk_extra,
+        chunk_header,
+        &outgoing_receipts_root,
+    )
+}
+
+/// Validate that all next chunk information matches previous chunk extra.
+pub fn validate_chunk_with_chunk_extra_and_receipts_root(
+    prev_chunk_extra: &ChunkExtra,
+    chunk_header: &ShardChunkHeader,
+    outgoing_receipts_root: &CryptoHash,
 ) -> Result<(), Error> {
     if *prev_chunk_extra.state_root() != chunk_header.prev_state_root() {
         return Err(Error::InvalidStateRoot);
@@ -140,19 +165,7 @@ pub fn validate_chunk_with_chunk_extra(
         return Err(Error::InvalidBalanceBurnt);
     }
 
-    let outgoing_receipts = chain_store.get_outgoing_receipts_for_shard(
-        epoch_manager,
-        *prev_block_hash,
-        chunk_header.shard_id(),
-        prev_chunk_height_included,
-    )?;
-    let outgoing_receipts_hashes = {
-        let shard_layout = epoch_manager.get_shard_layout_from_prev_block(prev_block_hash)?;
-        Chain::build_receipts_hashes(&outgoing_receipts, &shard_layout)
-    };
-    let (outgoing_receipts_root, _) = merklize(&outgoing_receipts_hashes);
-
-    if outgoing_receipts_root != chunk_header.prev_outgoing_receipts_root() {
+    if outgoing_receipts_root != &chunk_header.prev_outgoing_receipts_root() {
         return Err(Error::InvalidReceiptsProof);
     }
 

chain/chunks/src/lib.rs

@@ -111,8 +111,8 @@ use near_primitives::merkle::{verify_path, MerklePath};
 use near_primitives::receipt::Receipt;
 use near_primitives::sharding::{
     ChunkHash, EncodedShardChunk, EncodedShardChunkBody, PartialEncodedChunk,
-    PartialEncodedChunkPart, PartialEncodedChunkV2, ReceiptList, ReceiptProof, ReedSolomonWrapper,
-    ShardChunk, ShardChunkHeader, ShardProof,
+    PartialEncodedChunkPart, PartialEncodedChunkV2, ReceiptProof, ReedSolomonWrapper, ShardChunk,
+    ShardChunkHeader, ShardProof,
 };
 use near_primitives::transaction::SignedTransaction;
 use near_primitives::types::validator_stake::ValidatorStake;
@@ -1469,14 +1469,7 @@ impl ShardsManager {
             // https://github.com/near/nearcore/issues/5885
             // we can't simply use prev_block_hash to check if the node tracks this shard or not
             // because prev_block_hash may not be ready
-            let shard_id = proof.1.to_shard_id;
-            let ReceiptProof(shard_receipts, receipt_proof) = proof;
-            let receipt_hash = CryptoHash::hash_borsh(ReceiptList(shard_id, shard_receipts));
-            if !verify_path(
-                header.prev_outgoing_receipts_root(),
-                &receipt_proof.proof,
-                &receipt_hash,
-            ) {
+            if !proof.verify_against_receipt_root(header.prev_outgoing_receipts_root()) {
                 byzantine_assert!(false);
                 return Err(Error::ChainError(near_chain::Error::InvalidReceiptsProof));
             }