Restore receipts after recent fix

Cargo.toml

@@ -45,7 +45,8 @@ members = [
     "tools/restaked",
     "tools/indexer/example",
     "tools/delay_detector",
-    "tools/storage-usage-delta-calculator"
+    "tools/storage-usage-delta-calculator",
+    "tools/restored-receipts-verifier"
 ]
 
 [dev-dependencies]
@@ -60,6 +61,7 @@ near-crypto = { path = "./core/crypto" }
 near-primitives = { path = "./core/primitives" }
 near-store = { path = "./core/store" }
 runtime-params-estimator = { path = "./runtime/runtime-params-estimator", default_features = false }
+restored-receipts-verifier = { path = "./tools/restored-receipts-verifier"}
 near-chain = { path = "./chain/chain" }
 
 node-runtime = { path = "./runtime/runtime" }
@@ -113,14 +115,15 @@ metric_recorder = ["neard/metric_recorder"]
 delay_detector = ["neard/delay_detector"]
 rosetta_rpc = ["neard/rosetta_rpc"]
 nightly_protocol = ["near-primitives/nightly_protocol", "near-jsonrpc/nightly_protocol", "testlib/nightly_protocol", "neard/nightly_protocol"]
-nightly_protocol_features = ["nightly_protocol", "neard/nightly_protocol_features", "protocol_feature_evm", "protocol_feature_block_header_v3", "protocol_feature_alt_bn128", "protocol_feature_add_account_versions", "protocol_feature_tx_size_limit", "protocol_feature_allow_create_account_on_delete", "testlib/nightly_protocol_features", "protocol_feature_fix_storage_usage"]
+nightly_protocol_features = ["nightly_protocol", "neard/nightly_protocol_features", "protocol_feature_evm", "protocol_feature_block_header_v3", "protocol_feature_alt_bn128", "protocol_feature_add_account_versions", "protocol_feature_tx_size_limit", "protocol_feature_allow_create_account_on_delete", "testlib/nightly_protocol_features", "protocol_feature_fix_storage_usage", "protocol_feature_restore_receipts_after_fix"]
 protocol_feature_evm = ["neard/protocol_feature_evm", "testlib/protocol_feature_evm", "runtime-params-estimator/protocol_feature_evm"]
 protocol_feature_alt_bn128 = ["neard/protocol_feature_alt_bn128", "testlib/protocol_feature_alt_bn128", "runtime-params-estimator/protocol_feature_alt_bn128"]
 protocol_feature_block_header_v3 = ["near-primitives/protocol_feature_block_header_v3", "near-chain/protocol_feature_block_header_v3", "neard/protocol_feature_block_header_v3"]
 protocol_feature_tx_size_limit = ["near-primitives/protocol_feature_tx_size_limit", "node-runtime/protocol_feature_tx_size_limit", "neard/protocol_feature_tx_size_limit"]
 protocol_feature_allow_create_account_on_delete = ["testlib/protocol_feature_allow_create_account_on_delete", "near-primitives/protocol_feature_allow_create_account_on_delete", "node-runtime/protocol_feature_allow_create_account_on_delete", "neard/protocol_feature_allow_create_account_on_delete"]
 protocol_feature_add_account_versions = ["near-primitives/protocol_feature_add_account_versions"]
 protocol_feature_fix_storage_usage = ["near-primitives/protocol_feature_fix_storage_usage", "neard/protocol_feature_fix_storage_usage", "node-runtime/protocol_feature_fix_storage_usage"]
+protocol_feature_restore_receipts_after_fix = ["near-primitives/protocol_feature_restore_receipts_after_fix", "near-chain/protocol_feature_restore_receipts_after_fix", "neard/protocol_feature_restore_receipts_after_fix", "node-runtime/protocol_feature_restore_receipts_after_fix", "restored-receipts-verifier/protocol_feature_restore_receipts_after_fix"]
 
 # enable this to build neard with wasmer 1.0 runner
 # now if none of wasmer0_default, wasmer1_default or wasmtime_default is enabled, wasmer0 would be default

chain/chain/Cargo.toml

@@ -42,5 +42,6 @@ delay_detector = ["delay-detector"]
 no_cache = ["near-store/no_cache"]
 protocol_feature_evm = ["near-primitives/protocol_feature_evm", "near-chain-configs/protocol_feature_evm"]
 protocol_feature_block_header_v3 = []
-nightly_protocol_features = ["nightly_protocol", "protocol_feature_block_header_v3"]
+protocol_feature_restore_receipts_after_fix = []
+nightly_protocol_features = ["nightly_protocol", "protocol_feature_block_header_v3", "protocol_feature_restore_receipts_after_fix"]
 nightly_protocol = []

chain/chain/src/chain.rs

@@ -11,20 +11,8 @@ use rand::seq::SliceRandom;
 use rand::SeedableRng;
 use tracing::{debug, error, info, warn};
 
-use crate::lightclient::get_epoch_block_producers_view;
-use crate::missing_chunks::{BlockLike, MissingChunksPool};
-use crate::store::{ChainStore, ChainStoreAccess, ChainStoreUpdate, GCMode};
-use crate::types::{
-    AcceptedBlock, ApplyTransactionResult, Block, BlockEconomicsConfig, BlockHeader,
-    BlockHeaderInfo, BlockStatus, ChainGenesis, Provenance, RuntimeAdapter,
-};
-use crate::validate::{
-    validate_challenge, validate_chunk_proofs, validate_chunk_with_chunk_extra,
-    validate_transactions_order,
-};
-use crate::{byzantine_assert, create_light_client_block_view, Doomslug};
-use crate::{metrics, DoomslugThresholdMode};
-
+#[cfg(feature = "delay_detector")]
+use delay_detector::DelayDetector;
 use near_chain_primitives::error::{Error, ErrorKind, LogTransientStorageError};
 use near_primitives::block::{genesis_chunks, Tip};
 use near_primitives::challenge::{
@@ -62,8 +50,20 @@ use near_primitives::views::{
 };
 use near_store::{ColState, ColStateHeaders, ColStateParts, ShardTries, StoreUpdate};
 
-#[cfg(feature = "delay_detector")]
-use delay_detector::DelayDetector;
+use crate::lightclient::get_epoch_block_producers_view;
+use crate::migrations::check_if_block_is_first_with_chunk_of_version;
+use crate::missing_chunks::{BlockLike, MissingChunksPool};
+use crate::store::{ChainStore, ChainStoreAccess, ChainStoreUpdate, GCMode};
+use crate::types::{
+    AcceptedBlock, ApplyTransactionResult, Block, BlockEconomicsConfig, BlockHeader,
+    BlockHeaderInfo, BlockStatus, ChainGenesis, Provenance, RuntimeAdapter,
+};
+use crate::validate::{
+    validate_challenge, validate_chunk_proofs, validate_chunk_with_chunk_extra,
+    validate_transactions_order,
+};
+use crate::{byzantine_assert, create_light_client_block_view, Doomslug};
+use crate::{metrics, DoomslugThresholdMode};
 
 /// Maximum number of orphans chain can store.
 pub const MAX_ORPHAN_SIZE: usize = 1024;
@@ -2688,6 +2688,12 @@ impl<'a> ChainUpdate<'a> {
             Some(&block.hash()),
         )?;
         let prev_chunk_inner = prev_chunk.cloned_header().take_inner();
+        let is_first_block_with_chunk_of_version = check_if_block_is_first_with_chunk_of_version(
+            &mut self.chain_store_update,
+            self.runtime_adapter.as_ref(),
+            &prev_block.hash(),
+            chunk_shard_id,
+        )?;
         let apply_result = self
             .runtime_adapter
             .apply_transactions_with_optional_storage_proof(
@@ -2706,6 +2712,7 @@ impl<'a> ChainUpdate<'a> {
                 *block.header().random_value(),
                 true,
                 true,
+                is_first_block_with_chunk_of_version,
             )
             .unwrap();
         let partial_state = apply_result.proof.unwrap().nodes;
@@ -2792,15 +2799,13 @@ impl<'a> ChainUpdate<'a> {
                             Err(err) => err,
                         }
                     })?;
-
                     let receipt_proof_response: Vec<ReceiptProofResponse> =
                         self.chain_store_update.get_incoming_receipts_for_shard(
                             shard_id,
                             *block.hash(),
                             prev_chunk_header.height_included(),
                         )?;
                     let receipts = collect_receipts_from_response(&receipt_proof_response);
-
                     let chunk = self
                         .chain_store_update
                         .get_chunk_clone_from_header(&chunk_header.clone())?;
@@ -2836,6 +2841,18 @@ impl<'a> ChainUpdate<'a> {
                     let chunk_inner = chunk.cloned_header().take_inner();
                     let gas_limit = chunk_inner.gas_limit();
 
+                    // This variable is responsible for checking to which block we can apply receipts previously lost in apply_chunks
+                    // (see https://github.com/near/nearcore/pull/4248/)
+                    // We take the first block with existing chunk in the first epoch in which protocol feature
+                    // RestoreReceiptsAfterFix was enabled, and put the restored receipts there.
+                    let is_first_block_with_chunk_of_version =
+                        check_if_block_is_first_with_chunk_of_version(
+                            &mut self.chain_store_update,
+                            self.runtime_adapter.as_ref(),
+                            &prev_block.hash(),
+                            shard_id,
+                        )?;
+
                     // Apply transactions and receipts.
                     let apply_result = self
                         .runtime_adapter
@@ -2854,6 +2871,7 @@ impl<'a> ChainUpdate<'a> {
                             &block.header().challenges_result(),
                             *block.header().random_value(),
                             true,
+                            is_first_block_with_chunk_of_version,
                         )
                         .map_err(|e| ErrorKind::Other(e.to_string()))?;
 
@@ -2909,26 +2927,14 @@ impl<'a> ChainUpdate<'a> {
                             &block.header().challenges_result(),
                             *block.header().random_value(),
                             false,
+                            false,
                         )
                         .map_err(|e| ErrorKind::Other(e.to_string()))?;
 
                     self.chain_store_update.save_trie_changes(apply_result.trie_changes);
                     *new_extra.state_root_mut() = apply_result.new_root;
 
                     self.chain_store_update.save_chunk_extra(&block.hash(), shard_id, new_extra);
-
-                    if !apply_result.outcomes.is_empty() {
-                        // debug_assert!(false);
-                        // Remove in next release
-                        let (_, outcome_paths) =
-                            ApplyTransactionResult::compute_outcomes_proof(&apply_result.outcomes);
-                        self.chain_store_update.save_outcomes_with_proofs(
-                            &block.hash(),
-                            shard_id,
-                            apply_result.outcomes,
-                            outcome_paths,
-                        );
-                    }
                 }
             }
         }
@@ -3602,6 +3608,13 @@ impl<'a> ChainUpdate<'a> {
 
         let chunk_header = chunk.cloned_header();
         let gas_limit = chunk_header.gas_limit();
+        let is_first_block_with_chunk_of_version = check_if_block_is_first_with_chunk_of_version(
+            &mut self.chain_store_update,
+            self.runtime_adapter.as_ref(),
+            &chunk_header.prev_block_hash(),
+            shard_id,
+        )?;
+
         let apply_result = self.runtime_adapter.apply_transactions(
             shard_id,
             &chunk_header.prev_state_root(),
@@ -3617,6 +3630,7 @@ impl<'a> ChainUpdate<'a> {
             &block_header.challenges_result(),
             *block_header.random_value(),
             true,
+            is_first_block_with_chunk_of_version,
         )?;
 
         let (outcome_root, outcome_proofs) =
@@ -3696,6 +3710,7 @@ impl<'a> ChainUpdate<'a> {
             &block_header.challenges_result(),
             *block_header.random_value(),
             false,
+            false,
         )?;
 
         self.chain_store_update.save_trie_changes(apply_result.trie_changes);

chain/chain/src/lib.rs

@@ -5,16 +5,16 @@ pub use chain::{collect_receipts, Chain, MAX_ORPHAN_SIZE};
 pub use doomslug::{Doomslug, DoomslugBlockProductionReadiness, DoomslugThresholdMode};
 pub use lightclient::{create_light_client_block_view, get_epoch_block_producers_view};
 pub use near_chain_primitives::{self, Error, ErrorKind};
+pub use near_primitives::receipt::ReceiptResult;
 pub use store::{ChainStore, ChainStoreAccess, ChainStoreUpdate};
 pub use store_validator::{ErrorMessage, StoreValidator};
-pub use types::{
-    Block, BlockHeader, BlockStatus, ChainGenesis, Provenance, ReceiptResult, RuntimeAdapter,
-};
+pub use types::{Block, BlockHeader, BlockStatus, ChainGenesis, Provenance, RuntimeAdapter};
 
 pub mod chain;
 mod doomslug;
 mod lightclient;
 mod metrics;
+pub mod migrations;
 pub mod missing_chunks;
 mod store;
 pub mod store_validator;

chain/chain/src/migrations.rs

@@ -0,0 +1,38 @@
+use crate::store::ChainStoreAccess;
+use crate::types::RuntimeAdapter;
+use near_chain_primitives::error::Error;
+use near_primitives::hash::CryptoHash;
+use near_primitives::types::ShardId;
+
+/// Check that epoch of block with given prev_block_hash is the first one with current protocol version.
+fn is_first_epoch_with_protocol_version(
+    runtime_adapter: &dyn RuntimeAdapter,
+    prev_block_hash: &CryptoHash,
+) -> Result<bool, Error> {
+    let prev_epoch_id = runtime_adapter.get_prev_epoch_id_from_prev_block(prev_block_hash)?;
+    let epoch_id = runtime_adapter.get_epoch_id_from_prev_block(prev_block_hash)?;
+    let protocol_version = runtime_adapter.get_epoch_protocol_version(&epoch_id)?;
+    let prev_epoch_protocol_version = runtime_adapter.get_epoch_protocol_version(&prev_epoch_id)?;
+    Ok(protocol_version != prev_epoch_protocol_version)
+}
+
+/// Check that block is the first one with existing chunk for the given shard in the chain with its protocol version.
+/// We assume that current block contain the chunk for shard with the given id.
+pub fn check_if_block_is_first_with_chunk_of_version(
+    chain_store: &mut dyn ChainStoreAccess,
+    runtime_adapter: &dyn RuntimeAdapter,
+    prev_block_hash: &CryptoHash,
+    shard_id: ShardId,
+) -> Result<bool, Error> {
+    // Check that block belongs to the first epoch with current protocol version
+    // to avoid get_epoch_id_of_last_block_with_chunk call in the opposite case
+    if is_first_epoch_with_protocol_version(runtime_adapter, prev_block_hash)? {
+        // Compare only epochs because we already know that current epoch is the first one with current protocol version
+        let prev_epoch_id =
+            chain_store.get_epoch_id_of_last_block_with_chunk(prev_block_hash, shard_id)?;
+        let epoch_id = runtime_adapter.get_epoch_id_from_prev_block(prev_block_hash)?;
+        Ok(prev_epoch_id != epoch_id)
+    } else {
+        Ok(false)
+    }
+}

chain/chain/src/store.rs

@@ -13,7 +13,7 @@ use near_primitives::block::{Approval, Tip};
 use near_primitives::errors::InvalidTxError;
 use near_primitives::hash::CryptoHash;
 use near_primitives::merkle::{MerklePath, PartialMerkleTree};
-use near_primitives::receipt::Receipt;
+use near_primitives::receipt::{Receipt, ReceiptResult};
 use near_primitives::sharding::{
     ChunkHash, EncodedShardChunk, PartialEncodedChunk, ReceiptProof, ShardChunk, ShardChunkHeader,
     StateSyncInfo,
@@ -48,8 +48,8 @@ use near_store::{
     TAIL_KEY,
 };
 
+use crate::byzantine_assert;
 use crate::types::{Block, BlockHeader, LatestKnown};
-use crate::{byzantine_assert, ReceiptResult};
 
 /// lru cache size
 #[cfg(not(feature = "no_cache"))]
@@ -278,6 +278,22 @@ pub trait ChainStoreAccess {
             Ok(self.get_block_header(hash)?.height())
         }
     }
+
+    /// Get epoch id of the last block with existing chunk for the given shard id.
+    fn get_epoch_id_of_last_block_with_chunk(
+        &mut self,
+        hash: &CryptoHash,
+        shard_id: ShardId,
+    ) -> Result<EpochId, Error> {
+        let mut candidate_hash = *hash;
+        loop {
+            let block_header = self.get_block_header(&candidate_hash)?;
+            if block_header.chunk_mask()[shard_id as usize] {
+                break Ok(block_header.epoch_id().clone());
+            }
+            candidate_hash = *block_header.prev_hash();
+        }
+    }
 }
 
 /// All chain-related database operations.

chain/chain/src/test_utils.rs

@@ -604,6 +604,7 @@ impl RuntimeAdapter for KeyValueRuntime {
         _random_seed: CryptoHash,
         generate_storage_proof: bool,
         _is_new_chunk: bool,
+        _is_first_block_with_chunk_of_version: bool,
     ) -> Result<ApplyTransactionResult, Error> {
         assert!(!generate_storage_proof);
         let mut tx_results = vec![];
@@ -783,6 +784,7 @@ impl RuntimeAdapter for KeyValueRuntime {
         _challenges: &ChallengesResult,
         _random_value: CryptoHash,
         _is_new_chunk: bool,
+        _is_first_block_with_chunk_of_version: bool,
     ) -> Result<ApplyTransactionResult, Error> {
         unimplemented!();
     }
@@ -1098,6 +1100,22 @@ impl RuntimeAdapter for KeyValueRuntime {
     fn get_protocol_config(&self, _epoch_id: &EpochId) -> Result<ProtocolConfig, Error> {
         unreachable!("get_protocol_config should not be called in KeyValueRuntime");
     }
+
+    fn get_prev_epoch_id_from_prev_block(
+        &self,
+        prev_block_hash: &CryptoHash,
+    ) -> Result<EpochId, Error> {
+        let mut candidate_hash = prev_block_hash.clone();
+        loop {
+            let header = self
+                .get_block_header(&candidate_hash)?
+                .ok_or_else(|| ErrorKind::DBNotFoundErr(to_base(&candidate_hash)))?;
+            candidate_hash = header.prev_hash().clone();
+            if self.is_next_block_epoch_start(&candidate_hash)? {
+                break Ok(self.get_epoch_and_valset(candidate_hash)?.0);
+            }
+        }
+    }
 }
 
 pub fn setup() -> (Chain, Arc<KeyValueRuntime>, Arc<InMemoryValidatorSigner>) {