vm: AnyCache (#10825)

`AnyCache` is a part of the now-expanded concept of the
`ContractRuntimeCache`. It allows runtimes to put anything they would
like into it, so long as this cache is keyed by `CryptoKey`.

Initially I was in a lot of pain as intermediate drafts have required me
to add a `unsafe impl Sync for UniversalArtifact` which I'm not sure I
could possibly prove. However in the end I ended up being able to avoid
it entirely by giving out `AnyCache` by reference only. Lovely! -- EDIT:
crappa, looks like I did something wrong testing this locally T_T Sync
appears to remain a requirement.

Note that only NearVM uses this cache currently. I don't see a good
reason to port this code back to the old runtimes.

Supersedes #10787
Co-authored-by: Longarithm <the.aleksandr.logunov@gmail.com>

Cargo.toml

@@ -137,6 +137,7 @@ borsh = { version = "1.0.0", features = ["derive", "rc"] }
 bs58 = "0.4"
 bytes = "1"
 bytesize = { version = "1.1", features = ["serde"] }
+cov-mark = "2.0.0-pre.1"
 cargo_metadata = "0.14.1"
 cc = "1.0"
 cfg-if = "1.0"

chain/chain/src/runtime/mod.rs

@@ -43,7 +43,7 @@ use near_store::{
     TrieUpdate, WrappedTrieChanges, COLD_HEAD_KEY,
 };
 use near_vm_runner::ContractCode;
-use near_vm_runner::{precompile_contract, CompiledContractCache, FilesystemCompiledContractCache};
+use near_vm_runner::{precompile_contract, ContractRuntimeCache, FilesystemContractRuntimeCache};
 use node_runtime::adapter::ViewRuntimeAdapter;
 use node_runtime::state_viewer::TrieViewer;
 use node_runtime::{
@@ -68,7 +68,7 @@ pub struct NightshadeRuntime {
     runtime_config_store: RuntimeConfigStore,
 
     store: Store,
-    compiled_contract_cache: Box<dyn CompiledContractCache>,
+    compiled_contract_cache: Box<dyn ContractRuntimeCache>,
     tries: ShardTries,
     trie_viewer: TrieViewer,
     pub runtime: Runtime,
@@ -80,7 +80,7 @@ pub struct NightshadeRuntime {
 impl NightshadeRuntime {
     pub fn new(
         store: Store,
-        compiled_contract_cache: Box<dyn CompiledContractCache>,
+        compiled_contract_cache: Box<dyn ContractRuntimeCache>,
         genesis_config: &GenesisConfig,
         epoch_manager: Arc<EpochManagerHandle>,
         trie_viewer_state_size_limit: Option<u64>,
@@ -133,7 +133,7 @@ impl NightshadeRuntime {
     pub fn test_with_runtime_config_store(
         home_dir: &Path,
         store: Store,
-        compiled_contract_cache: Box<dyn CompiledContractCache>,
+        compiled_contract_cache: Box<dyn ContractRuntimeCache>,
         genesis_config: &GenesisConfig,
         epoch_manager: Arc<EpochManagerHandle>,
         runtime_config_store: RuntimeConfigStore,
@@ -161,7 +161,7 @@ impl NightshadeRuntime {
     pub fn test_with_trie_config(
         home_dir: &Path,
         store: Store,
-        compiled_contract_cache: Box<dyn CompiledContractCache>,
+        compiled_contract_cache: Box<dyn ContractRuntimeCache>,
         genesis_config: &GenesisConfig,
         epoch_manager: Arc<EpochManagerHandle>,
         trie_config: TrieConfig,
@@ -195,7 +195,7 @@ impl NightshadeRuntime {
         Self::test_with_runtime_config_store(
             home_dir,
             store,
-            FilesystemCompiledContractCache::new(home_dir, None::<&str>)
+            FilesystemContractRuntimeCache::with_memory_cache(home_dir, None::<&str>, 1)
                 .expect("filesystem contract cache")
                 .handle(),
             genesis_config,
@@ -463,7 +463,7 @@ impl NightshadeRuntime {
         .entered();
         let protocol_version = self.epoch_manager.get_epoch_protocol_version(epoch_id)?;
         let runtime_config = self.runtime_config_store.get_config(protocol_version);
-        let compiled_contract_cache: Option<Box<dyn CompiledContractCache>> =
+        let compiled_contract_cache: Option<Box<dyn ContractRuntimeCache>> =
             Some(Box::new(self.compiled_contract_cache.handle()));
         // Execute precompile_contract in parallel but prevent it from using more than half of all
         // threads so that node will still function normally.

chain/chain/src/runtime/tests.rs

@@ -198,7 +198,7 @@ impl TestEnv {
             if config.zero_fees { RuntimeConfigStore::free() } else { RuntimeConfigStore::test() };
 
         let compiled_contract_cache =
-            FilesystemCompiledContractCache::new(&dir.as_ref(), None::<&str>).unwrap();
+            FilesystemContractRuntimeCache::new(&dir.as_ref(), None::<&str>).unwrap();
 
         initialize_genesis_state(store.clone(), &genesis, Some(dir.path()));
         let epoch_manager = EpochManager::new_arc_handle(store.clone(), &genesis.config);
@@ -1461,7 +1461,7 @@ fn test_genesis_hash() {
     let runtime = NightshadeRuntime::test_with_runtime_config_store(
         tempdir.path(),
         store.clone(),
-        FilesystemCompiledContractCache::new(tempdir.path(), None::<&str>)
+        FilesystemContractRuntimeCache::new(tempdir.path(), None::<&str>)
             .expect("filesystem contract cache")
             .handle(),
         &genesis.config,

chain/client/src/test_utils/test_env_builder.rs

@@ -20,7 +20,7 @@ use near_primitives::types::{AccountId, NumShards};
 use near_store::config::StateSnapshotType;
 use near_store::test_utils::create_test_store;
 use near_store::{NodeStorage, ShardUId, Store, StoreConfig, TrieConfig};
-use near_vm_runner::{CompiledContractCache, FilesystemCompiledContractCache};
+use near_vm_runner::{ContractRuntimeCache, FilesystemContractRuntimeCache};
 use std::collections::HashMap;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -48,7 +48,7 @@ pub struct TestEnvBuilder {
     validators: Vec<AccountId>,
     home_dirs: Option<Vec<PathBuf>>,
     stores: Option<Vec<Store>>,
-    contract_caches: Option<Vec<Box<dyn CompiledContractCache>>>,
+    contract_caches: Option<Vec<Box<dyn ContractRuntimeCache>>>,
     epoch_managers: Option<Vec<EpochManagerKind>>,
     shard_trackers: Option<Vec<ShardTracker>>,
     runtimes: Option<Vec<Arc<dyn RuntimeAdapter>>>,
@@ -169,7 +169,7 @@ impl TestEnvBuilder {
         self
     }
 
-    pub fn contract_caches<C: CompiledContractCache>(
+    pub fn contract_caches<C: ContractRuntimeCache>(
         mut self,
         caches: impl IntoIterator<Item = C>,
     ) -> Self {
@@ -217,7 +217,7 @@ impl TestEnvBuilder {
             return self;
         }
         let count = self.clients.len();
-        self.contract_caches((0..count).map(|_| FilesystemCompiledContractCache::test().unwrap()))
+        self.contract_caches((0..count).map(|_| FilesystemContractRuntimeCache::test().unwrap()))
     }
 
     /// Specifies custom EpochManagerHandle for each client.  This allows us to
@@ -310,7 +310,7 @@ impl TestEnvBuilder {
         nightshade_runtime_creator: impl Fn(
             PathBuf,
             Store,
-            Box<dyn CompiledContractCache>,
+            Box<dyn ContractRuntimeCache>,
             Arc<EpochManagerHandle>,
             RuntimeConfigStore,
             TrieConfig,

core/primitives/src/views.rs

@@ -98,7 +98,7 @@ pub struct ViewApplyState {
     /// Current Protocol version when we apply the state transition
     pub current_protocol_version: ProtocolVersion,
     /// Cache for compiled contracts.
-    pub cache: Option<Box<dyn near_vm_runner::CompiledContractCache>>,
+    pub cache: Option<Box<dyn near_vm_runner::ContractRuntimeCache>>,
 }
 
 impl From<&Account> for AccountView {

core/store/src/columns.rs

@@ -235,7 +235,7 @@ pub enum DBCol {
     /// - *Rows*: receipt (CryptoHash)
     /// - *Column type*: Receipt
     Receipts,
-    /// Precompiled machine code of the contract, used by StoreCompiledContractCache.
+    /// Precompiled machine code of the contract, used by StoreContractRuntimeCache.
     /// - *Rows*: ContractCacheKey or code hash (not sure)
     /// - *Column type*: near-vm-runner CacheRecord
     CachedContractCode,

core/store/src/lib.rs

@@ -30,7 +30,7 @@ use near_primitives::receipt::{
 pub use near_primitives::shard_layout::ShardUId;
 use near_primitives::trie_key::{trie_key_parsers, TrieKey};
 use near_primitives::types::{AccountId, BlockHeight, StateRoot};
-use near_vm_runner::{CompiledContract, CompiledContractCache, ContractCode};
+use near_vm_runner::{CompiledContract, ContractCode, ContractRuntimeCache};
 use once_cell::sync::Lazy;
 use std::fs::File;
 use std::path::Path;
@@ -953,11 +953,11 @@ where
 }
 
 #[derive(Clone)]
-pub struct StoreCompiledContractCache {
+pub struct StoreContractRuntimeCache {
     db: Arc<dyn Database>,
 }
 
-impl StoreCompiledContractCache {
+impl StoreContractRuntimeCache {
     pub fn new(store: &Store) -> Self {
         Self { db: store.storage.clone() }
     }
@@ -967,11 +967,11 @@ impl StoreCompiledContractCache {
 /// We store contracts in VM-specific format in DBCol::CachedContractCode.
 /// Key must take into account VM being used and its configuration, so that
 /// we don't cache non-gas metered binaries, for example.
-impl CompiledContractCache for StoreCompiledContractCache {
+impl ContractRuntimeCache for StoreContractRuntimeCache {
     #[tracing::instrument(
         level = "trace",
         target = "store",
-        "StoreCompiledContractCache::put",
+        "StoreContractRuntimeCache::put",
         skip_all,
         fields(key = key.to_string(), value.len = value.debug_len()),
     )]
@@ -992,7 +992,7 @@ impl CompiledContractCache for StoreCompiledContractCache {
     #[tracing::instrument(
         level = "trace",
         target = "store",
-        "StoreCompiledContractCache::get",
+        "StoreContractRuntimeCache::get",
         skip_all,
         fields(key = key.to_string()),
     )]
@@ -1008,7 +1008,7 @@ impl CompiledContractCache for StoreCompiledContractCache {
         self.db.get_raw_bytes(DBCol::CachedContractCode, key.as_ref()).map(|entry| entry.is_some())
     }
 
-    fn handle(&self) -> Box<dyn CompiledContractCache> {
+    fn handle(&self) -> Box<dyn ContractRuntimeCache> {
         Box::new(self.clone())
     }
 }
@@ -1126,14 +1126,14 @@ mod tests {
         test_iter_order_impl(crate::test_utils::create_test_store());
     }
 
-    /// Check StoreCompiledContractCache implementation.
+    /// Check StoreContractRuntimeCache implementation.
     #[test]
     fn test_store_compiled_contract_cache() {
-        use near_vm_runner::{CompiledContract, CompiledContractCache};
+        use near_vm_runner::{CompiledContract, ContractRuntimeCache};
         use std::str::FromStr;
 
         let store = crate::test_utils::create_test_store();
-        let cache = super::StoreCompiledContractCache::new(&store);
+        let cache = super::StoreContractRuntimeCache::new(&store);
         let key = CryptoHash::from_str("75pAU4CJcp8Z9eoXcL6pSU8sRK5vn3NEpgvUrzZwQtr3").unwrap();
 
         assert_eq!(None, cache.get(&key).unwrap());

deny.toml

@@ -98,9 +98,7 @@ skip = [
     { name = "windows-targets", version = "=0.42.1" },
 
     # ed25519-dalek uses older versions of rand and rand_core
-    { name = "rand", version = "=0.7.3" },
     { name = "rand_core", version = "=0.5.1" },
-    { name = "rand_chacha", version = "=0.2.2" },
     { name = "getrandom", version = "=0.1.16" },
 
     # criterion and criterion-plot use conflicting versions
@@ -137,9 +135,6 @@ skip = [
     # rust-s3 is using an old version of smartstring
     { name = "smartstring", version = "=0.2.10" },
 
-    # validator 0.12 ~ 0.16 is still using an old version of idna
-    { name = "idna", version = "=0.2.3" },
-
     # zeropool-bn uses borsh 0.9
     { name = "borsh", version = "=0.9.3" },
     { name = "borsh-derive", version = "=0.9.3" },
@@ -159,4 +154,7 @@ skip = [
     # cloud-storage
     { name = "base64", version = "=0.12.3" },
     { name = "num-bigint", version = "=0.2.6" },
+
+    # Everything depends on this...
+    { name = "lru", version = "=0.7.8" },
 ]

integration-tests/src/tests/client/features/multinode_test_loop_example.rs

@@ -61,8 +61,8 @@ use near_store::config::StateSnapshotType;
 use near_store::genesis::initialize_genesis_state;
 use near_store::test_utils::create_test_store;
 use near_store::TrieConfig;
-use near_vm_runner::CompiledContractCache;
-use near_vm_runner::FilesystemCompiledContractCache;
+use near_vm_runner::ContractRuntimeCache;
+use near_vm_runner::FilesystemContractRuntimeCache;
 use nearcore::NightshadeRuntime;
 use std::collections::HashMap;
 use std::path::Path;
@@ -229,7 +229,7 @@ fn test_client_with_multi_test_loop() {
             builder.sender().for_index(idx).into_sender(),
         )));
         let home_dir = Path::new(".");
-        let contract_cache = FilesystemCompiledContractCache::new(home_dir, None::<&str>)
+        let contract_cache = FilesystemContractRuntimeCache::new(home_dir, None::<&str>)
             .expect("filesystem contract cache")
             .handle();
         let runtime_adapter = NightshadeRuntime::test_with_trie_config(