diff --git a/program-runtime/src/loaded_programs.rs b/program-runtime/src/loaded_programs.rs index dc6dcd9b03c798..bf6ae10b069f0f 100644 --- a/program-runtime/src/loaded_programs.rs +++ b/program-runtime/src/loaded_programs.rs @@ -145,14 +145,34 @@ pub struct LoadedProgram { #[derive(Debug, Default)] pub struct Stats { +<<<<<<< HEAD pub hits: AtomicU64, +======= + /// a program was already in the cache + pub hits: AtomicU64, + /// a program was not found and loaded instead +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) pub misses: AtomicU64, pub evictions: HashMap, +<<<<<<< HEAD + pub insertions: AtomicU64, +======= + /// an unloaded program was loaded again (opposite of eviction) + pub reloads: AtomicU64, + /// a program was loaded or un/re/deployed pub insertions: AtomicU64, + /// a program was loaded but can not be extracted on its own fork anymore + pub lost_insertions: AtomicU64, + /// a program which was already in the cache was reloaded by mistake +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) pub replacements: AtomicU64, pub one_hit_wonders: AtomicU64, pub prunes_orphan: AtomicU64, +<<<<<<< HEAD pub prunes_expired: AtomicU64, +======= + /// a program got pruned because it was not recompiled for the next epoch +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) pub prunes_environment: AtomicU64, pub empty_entries: AtomicU64, } @@ -162,12 +182,13 @@ impl Stats { pub fn submit(&self, slot: Slot) { let hits = self.hits.load(Ordering::Relaxed); let misses = self.misses.load(Ordering::Relaxed); + let evictions: u64 = self.evictions.values().sum(); + let reloads = self.reloads.load(Ordering::Relaxed); let insertions = self.insertions.load(Ordering::Relaxed); + let lost_insertions = self.insertions.load(Ordering::Relaxed); let replacements = self.replacements.load(Ordering::Relaxed); let one_hit_wonders = self.one_hit_wonders.load(Ordering::Relaxed); - let evictions: u64 = self.evictions.values().sum(); let prunes_orphan = self.prunes_orphan.load(Ordering::Relaxed); - let prunes_expired = self.prunes_expired.load(Ordering::Relaxed); let prunes_environment = self.prunes_environment.load(Ordering::Relaxed); let empty_entries = self.empty_entries.load(Ordering::Relaxed); datapoint_info!( @@ -176,17 +197,18 @@ impl Stats { ("hits", hits, i64), ("misses", misses, i64), ("evictions", evictions, i64), + ("reloads", reloads, i64), ("insertions", insertions, i64), + ("lost_insertions", lost_insertions, i64), ("replacements", replacements, i64), ("one_hit_wonders", one_hit_wonders, i64), ("prunes_orphan", prunes_orphan, i64), - ("prunes_expired", prunes_expired, i64), ("prunes_environment", prunes_environment, i64), ("empty_entries", empty_entries, i64), ); debug!( - "Loaded Programs Cache Stats -- Hits: {}, Misses: {}, Evictions: {}, Insertions: {}, Replacements: {}, One-Hit-Wonders: {}, Prunes-Orphan: {}, Prunes-Expired: {}, Prunes-Environment: {}, Empty: {}", - hits, misses, evictions, insertions, replacements, one_hit_wonders, prunes_orphan, prunes_expired, prunes_environment, empty_entries + "Loaded Programs Cache Stats -- Hits: {}, Misses: {}, Evictions: {}, Reloads: {}, Insertions: {} Lost-Insertions: {}, Replacements: {}, One-Hit-Wonders: {}, Prunes-Orphan: {}, Prunes-Environment: {}, Empty: {}", + hits, misses, evictions, reloads, insertions, lost_insertions, replacements, one_hit_wonders, prunes_orphan, prunes_environment, empty_entries ); if log_enabled!(log::Level::Trace) && !self.evictions.is_empty() { let mut evictions = self.evictions.iter().collect::>(); @@ -640,9 +662,7 @@ impl LoadedPrograms { let index = slot_versions .iter() .position(|at| at.effective_slot >= entry.effective_slot); - if let Some((existing, entry_index)) = - index.and_then(|index| slot_versions.get(index).map(|value| (value, index))) - { + if let Some(existing) = index.and_then(|index| slot_versions.get_mut(index)) { if existing.deployment_slot == entry.deployment_slot && existing.effective_slot == entry.effective_slot { @@ -657,17 +677,19 @@ impl LoadedPrograms { existing.ix_usage_counter.load(Ordering::Relaxed), Ordering::Relaxed, ); - slot_versions.remove(entry_index); + self.stats.reloads.fetch_add(1, Ordering::Relaxed); } else if existing.is_tombstone() != entry.is_tombstone() { // Either the old entry is tombstone and the new one is not. // (Let's give the new entry a chance). // Or, the old entry is not a tombstone and the new one is a tombstone. // (Remove the old entry, as the tombstone makes it obsolete). - slot_versions.remove(entry_index); + self.stats.insertions.fetch_add(1, Ordering::Relaxed); } else { self.stats.replacements.fetch_add(1, Ordering::Relaxed); return (true, existing.clone()); } + *existing = entry.clone(); + return (false, entry); } } self.stats.insertions.fetch_add(1, Ordering::Relaxed); @@ -757,7 +779,6 @@ impl LoadedPrograms { // Remove expired if let Some(expiration) = entry.maybe_expiration_slot { if expiration <= new_root_slot { - self.stats.prunes_expired.fetch_add(1, Ordering::Relaxed); return false; } } @@ -831,11 +852,13 @@ impl LoadedPrograms { working_slot: &S, search_for: &mut Vec<(Pubkey, (LoadedProgramMatchCriteria, u64))>, loaded_programs_for_tx_batch: &mut LoadedProgramsForTxBatch, + is_first_round: bool, ) -> Option<(Pubkey, u64)> { let mut cooperative_loading_task = None; search_for.retain(|(key, (match_criteria, usage_count))| { if let Some(second_level) = self.entries.get_mut(key) { for entry in second_level.slot_versions.iter().rev() { +<<<<<<< HEAD let is_ancestor = if let Some(fork_graph) = &self.fork_graph { fork_graph .read() @@ -853,9 +876,16 @@ impl LoadedPrograms { working_slot.is_ancestor(entry.deployment_slot) }; +======= +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) if entry.deployment_slot <= self.latest_root_slot - || entry.deployment_slot == loaded_programs_for_tx_batch.slot - || is_ancestor + || matches!( + locked_fork_graph.relationship( + entry.deployment_slot, + loaded_programs_for_tx_batch.slot + ), + BlockRelation::Equal | BlockRelation::Ancestor + ) { let entry_to_return = if loaded_programs_for_tx_batch.slot >= entry.effective_slot { @@ -910,6 +940,7 @@ impl LoadedPrograms { } true }); +<<<<<<< HEAD self.stats .misses .fetch_add(search_for.len() as u64, Ordering::Relaxed); @@ -917,6 +948,18 @@ impl LoadedPrograms { loaded_programs_for_tx_batch.entries.len() as u64, Ordering::Relaxed, ); +======= + drop(locked_fork_graph); + if is_first_round { + self.stats + .misses + .fetch_add(search_for.len() as u64, Ordering::Relaxed); + self.stats.hits.fetch_add( + loaded_programs_for_tx_batch.entries.len() as u64, + Ordering::Relaxed, + ); + } +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) cooperative_loading_task } @@ -933,6 +976,20 @@ impl LoadedPrograms { Some((slot, std::thread::current().id())) ); second_level.cooperative_loading_lock = None; + // Check that it will be visible to our own fork once inserted + if loaded_program.deployment_slot > self.latest_root_slot + && !matches!( + self.fork_graph + .as_ref() + .unwrap() + .read() + .unwrap() + .relationship(loaded_program.deployment_slot, slot), + BlockRelation::Equal | BlockRelation::Ancestor + ) + { + self.stats.lost_insertions.fetch_add(1, Ordering::Relaxed); + } self.assign_program(key, loaded_program); self.loading_task_waiter.notify(); } @@ -1804,7 +1861,11 @@ mod tests { (program4, (LoadedProgramMatchCriteria::NoCriteria, 4)), ]; let mut extracted = LoadedProgramsForTxBatch::new(22, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(22), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 20, 22)); assert!(match_slot(&extracted, &program4, 0, 22)); @@ -1820,7 +1881,11 @@ mod tests { (program4, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(15, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(15), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 15)); assert!(match_slot(&extracted, &program2, 11, 15)); @@ -1843,7 +1908,11 @@ mod tests { (program4, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(18, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(18), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 18)); assert!(match_slot(&extracted, &program2, 11, 18)); @@ -1861,7 +1930,11 @@ mod tests { (program4, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(23, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(23), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 23)); assert!(match_slot(&extracted, &program2, 11, 23)); @@ -1879,7 +1952,11 @@ mod tests { (program4, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(11, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(11), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 11)); // program2 was updated at slot 11, but is not effective till slot 12. The result should contain a tombstone. @@ -1912,7 +1989,11 @@ mod tests { (program4, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(19, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(19), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 19)); assert!(match_slot(&extracted, &program2, 11, 19)); @@ -1930,7 +2011,11 @@ mod tests { (program4, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(21, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(21), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 21)); assert!(match_slot(&extracted, &program2, 11, 21)); @@ -1968,7 +2053,11 @@ mod tests { (program4, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(21, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(21), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) // Since the fork was pruned, we should not find the entry deployed at slot 20. assert!(match_slot(&extracted, &program1, 0, 21)); @@ -1985,7 +2074,11 @@ mod tests { (program4, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(27, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(27), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 27)); assert!(match_slot(&extracted, &program2, 11, 27)); @@ -2017,7 +2110,11 @@ mod tests { (program4, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(23, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(23), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 23)); assert!(match_slot(&extracted, &program2, 11, 23)); @@ -2072,7 +2169,11 @@ mod tests { (program3, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(12, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(12), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 12)); assert!(match_slot(&extracted, &program2, 11, 12)); @@ -2092,7 +2193,11 @@ mod tests { (program3, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(12, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(12), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program2, 11, 12)); @@ -2162,7 +2267,11 @@ mod tests { (program3, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(19, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(19), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 19)); assert!(match_slot(&extracted, &program2, 11, 19)); @@ -2176,7 +2285,11 @@ mod tests { (program3, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(27, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(27), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 27)); assert!(match_slot(&extracted, &program2, 11, 27)); @@ -2190,7 +2303,11 @@ mod tests { (program3, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(22, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(22), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 20, 22)); @@ -2254,7 +2371,11 @@ mod tests { (program3, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(12, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(12), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) // Program1 deployed at slot 11 should not be expired yet assert!(match_slot(&extracted, &program1, 11, 12)); @@ -2270,7 +2391,11 @@ mod tests { (program3, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(15, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(15), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program2, 11, 15)); @@ -2336,7 +2461,11 @@ mod tests { let mut missing = vec![(program1, (LoadedProgramMatchCriteria::NoCriteria, 1))]; let mut extracted = LoadedProgramsForTxBatch::new(20, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(20), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) // The cache should have the program deployed at slot 0 assert_eq!( @@ -2380,7 +2509,11 @@ mod tests { (program2, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(20, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(20), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 20)); assert!(match_slot(&extracted, &program2, 10, 20)); @@ -2390,7 +2523,11 @@ mod tests { (program2, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(6, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(6), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 5, 6)); assert!(match_missing(&missing, &program2, false)); @@ -2404,7 +2541,11 @@ mod tests { (program2, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(20, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(20), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 20)); assert!(match_slot(&extracted, &program2, 10, 20)); @@ -2414,7 +2555,11 @@ mod tests { (program2, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(6, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(6), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 6)); assert!(match_missing(&missing, &program2, false)); @@ -2428,7 +2573,11 @@ mod tests { (program2, (LoadedProgramMatchCriteria::NoCriteria, 1)), ]; let mut extracted = LoadedProgramsForTxBatch::new(20, cache.environments.clone()); +<<<<<<< HEAD cache.extract(&TestWorkingSlot(20), &mut missing, &mut extracted); +======= + cache.extract(&mut missing, &mut extracted, true); +>>>>>>> 5dd9609aea (Fix - LoadedPrograms statistics (#35026)) assert!(match_slot(&extracted, &program1, 0, 20)); assert!(match_missing(&missing, &program2, false)); diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index f30b368480f609..b53f037db4c3d3 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -5081,7 +5081,8 @@ impl Bank { // Lock the global cache. let mut loaded_programs_cache = self.loaded_programs_cache.write().unwrap(); // Initialize our local cache. - if loaded_programs_for_txs.is_none() { + let is_first_round = loaded_programs_for_txs.is_none(); + if is_first_round { loaded_programs_for_txs = Some(LoadedProgramsForTxBatch::new( self.slot, loaded_programs_cache @@ -5102,6 +5103,7 @@ impl Bank { self, &mut missing_programs, loaded_programs_for_txs.as_mut().unwrap(), + is_first_round, ); let task_waiter = Arc::clone(&loaded_programs_cache.loading_task_waiter); (program_to_load, task_waiter.cookie(), task_waiter)