From 27732c2f2b6c331a6a7b5d82fae331d1a0099e36 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Thu, 31 Mar 2022 19:29:00 +0200 Subject: [PATCH 1/5] Mmap the DepGraph instead of reading it. --- .../rustc_incremental/src/persist/load.rs | 2 +- .../rustc_query_system/src/dep_graph/mod.rs | 4 +- .../src/dep_graph/serialized.rs | 235 ++++++++++-------- compiler/rustc_serialize/src/lib.rs | 9 + compiler/rustc_serialize/src/opaque.rs | 54 ++++ 5 files changed, 197 insertions(+), 107 deletions(-) diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index 1c5fd91690230..78282f9a36270 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -204,7 +204,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { return LoadResult::DataOutOfDate; } - let dep_graph = SerializedDepGraph::decode(&mut decoder); + let dep_graph = SerializedDepGraph::decode(bytes); LoadResult::Ok { data: (dep_graph, prev_work_products) } } diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs index 342d95ca490ea..cd2e7f60ff76f 100644 --- a/compiler/rustc_query_system/src/dep_graph/mod.rs +++ b/compiler/rustc_query_system/src/dep_graph/mod.rs @@ -13,7 +13,7 @@ pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex}; use crate::ich::StableHashingContext; use rustc_data_structures::profiling::SelfProfilerRef; -use rustc_serialize::{opaque::FileEncoder, Encodable}; +use rustc_serialize::MmapSafe; use rustc_session::Session; use std::fmt; @@ -84,7 +84,7 @@ impl FingerprintStyle { } /// Describe the different families of dependency nodes. -pub trait DepKind: Copy + fmt::Debug + Eq + Hash + Send + Encodable + 'static { +pub trait DepKind: Copy + fmt::Debug + Eq + Hash + Send + MmapSafe + 'static { /// DepKind to use when incr. comp. is turned off. const NULL: Self; diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index 3b20ec70d73cb..ae0ec99bb35c2 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -16,11 +16,13 @@ use super::query::DepGraphQuery; use super::{DepKind, DepNode, DepNodeIndex}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::memmap::Mmap; +use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sync::Lock; -use rustc_index::vec::{Idx, IndexVec}; +use rustc_index::vec::IndexVec; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder}; -use rustc_serialize::{Decodable, Decoder, Encodable}; +use rustc_serialize::{Decodable, Encodable}; use smallvec::SmallVec; use std::convert::TryInto; @@ -34,138 +36,136 @@ rustc_index::newtype_index! { } /// Data for use when recompiling the **current crate**. -#[derive(Debug)] +/// +/// The DepGraph is backed on-disk and read on-demand through a Mmap. +/// The file layout is as follows. +/// +/// The DepGraph starts with the version header, handled by rustc_incremental. +/// It is followed by the concatenation of all node dependency information as: +/// - the query result fingerprint (size_of bytes) +/// - the number of dependency edges (4 bytes) +/// - the dependencies indices (array of 4-byte integers) +/// +/// Finding this information inside the file is handled by a "glossary" written at the end. +/// This glossary is an array of `(DepNode, u32)` pairs. This array is used to make the +/// correspondence between the `SerializedDepNodeIndex` (ie. the index into this array), +/// and the `DepNode`. The `u32` is the position of the dependency information (Fingerprint + +/// array of dependencies) inside the file. The glossary array is directly mmapped into `nodes`. +/// +/// The file finished with two `u64`, which are the number of entries in the glossary +/// and its position in the file. +/// +/// Graphically, we have: +/// beginning of nodes beginning of glossary ---------------+ +/// v v | +/// -------------------------------------------------------------------------------------------- +/// | HEADER | ... | Fingerprint | Length | Deps | ... | ... | DepNode | u32 | ... | u64 | u64 | +/// -------------------------------------------------------------------------------------------- +/// ^ | node +/// start for node i --------------------------------------+ count +/// +/// In order to recover an index from a DepNode, we populate a hash-map in `index`. pub struct SerializedDepGraph { - /// The set of all DepNodes in the graph - nodes: IndexVec>, - /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to - /// the DepNode at the same index in the nodes vector. - fingerprints: IndexVec, - /// For each DepNode, stores the list of edges originating from that - /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, - /// which holds the actual DepNodeIndices of the target nodes. - edge_list_indices: IndexVec, - /// A flattened list of all edge targets in the graph. Edge sources are - /// implicit in edge_list_indices. - edge_list_data: Vec, + /// The set of all DepNodes in the graph and their position in the mmap. + nodes: Option, u32)]>>, /// Reciprocal map to `nodes`. index: FxHashMap, SerializedDepNodeIndex>, } impl Default for SerializedDepGraph { fn default() -> Self { - SerializedDepGraph { - nodes: Default::default(), - fingerprints: Default::default(), - edge_list_indices: Default::default(), - edge_list_data: Default::default(), - index: Default::default(), - } + SerializedDepGraph { nodes: None, index: Default::default() } } } impl SerializedDepGraph { #[inline] - pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] { - let targets = self.edge_list_indices[source]; - &self.edge_list_data[targets.0 as usize..targets.1 as usize] + fn decoder_at(&self, dep_node_index: SerializedDepNodeIndex) -> Option> { + let nodes = self.nodes.as_ref()?; + let dep_node_index = dep_node_index.as_usize(); + let position = nodes[dep_node_index].1 as usize; + let data = &nodes.owner()[position..]; + let decoder = MemDecoder::new(data, 0); + Some(decoder) + } + + #[inline] + pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { + self.index.get(dep_node).cloned() } #[inline] pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode { - self.nodes[dep_node_index] + let dep_node_index = dep_node_index.as_usize(); + self.nodes.as_ref().unwrap()[dep_node_index].0 } #[inline] - pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { - self.index.get(dep_node).cloned() + pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint { + if let Some(decoder) = self.decoder_at(dep_node_index) { + let &fingerprint = unsafe { decoder.mmap_at::(0) }; + fingerprint + } else { + Fingerprint::ZERO + } } #[inline] pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option { - self.index.get(dep_node).map(|&node_index| self.fingerprints[node_index]) + let index = self.index.get(dep_node).cloned()?; + Some(self.fingerprint_by_index(index)) } #[inline] - pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint { - self.fingerprints[dep_node_index] + pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] { + // The encoder has checked that there is no padding there. + if let Some(decoder) = self.decoder_at(source) { + let position = std::mem::size_of::(); + let &length = unsafe { decoder.mmap_at::(position) }; + unsafe { + decoder.mmap_slice_at::(position + 4, length as usize) + } + } else { + &[] + } } pub fn node_count(&self) -> usize { self.index.len() } -} -impl<'a, K: DepKind + Decodable>> Decodable> - for SerializedDepGraph -{ - #[instrument(level = "debug", skip(d))] - fn decode(d: &mut MemDecoder<'a>) -> SerializedDepGraph { - let start_position = d.position(); + #[instrument(level = "debug", skip(mmap))] + pub fn decode(mmap: Mmap) -> SerializedDepGraph { + let data = mmap.as_ref(); - // The last 16 bytes are the node count and edge count. - debug!("position: {:?}", d.position()); - d.set_position(d.data.len() - 2 * IntEncodedWithFixedSize::ENCODED_SIZE); + // The last 16 bytes are the node count, edge count and nodes position. + let start_position = data.len() - 2 * IntEncodedWithFixedSize::ENCODED_SIZE; + let mut d = MemDecoder::new(data, start_position); debug!("position: {:?}", d.position()); - let node_count = IntEncodedWithFixedSize::decode(d).0 as usize; - let edge_count = IntEncodedWithFixedSize::decode(d).0 as usize; - debug!(?node_count, ?edge_count); + let node_count = IntEncodedWithFixedSize::decode(&mut d).0 as usize; + let nodes_position = IntEncodedWithFixedSize::decode(&mut d).0 as usize; + debug!(?node_count, ?nodes_position); - debug!("position: {:?}", d.position()); - d.set_position(start_position); - debug!("position: {:?}", d.position()); - - let mut nodes = IndexVec::with_capacity(node_count); - let mut fingerprints = IndexVec::with_capacity(node_count); - let mut edge_list_indices = IndexVec::with_capacity(node_count); - let mut edge_list_data = Vec::with_capacity(edge_count); - - for _index in 0..node_count { - let dep_node: DepNode = Decodable::decode(d); - let _i: SerializedDepNodeIndex = nodes.push(dep_node); - debug_assert_eq!(_i.index(), _index); - - let fingerprint: Fingerprint = Decodable::decode(d); - let _i: SerializedDepNodeIndex = fingerprints.push(fingerprint); - debug_assert_eq!(_i.index(), _index); - - // Deserialize edges -- sequence of DepNodeIndex - let len = d.read_usize(); - let start = edge_list_data.len().try_into().unwrap(); - for _ in 0..len { - let edge = Decodable::decode(d); - edge_list_data.push(edge); - } - let end = edge_list_data.len().try_into().unwrap(); - let _i: SerializedDepNodeIndex = edge_list_indices.push((start, end)); - debug_assert_eq!(_i.index(), _index); - } + let nodes = OwningRef::new(mmap).map(|mmap| { + let d = MemDecoder::new(mmap, nodes_position); + unsafe { d.mmap_slice_at::<(DepNode, u32)>(nodes_position, node_count) } + }); - let index: FxHashMap<_, _> = - nodes.iter_enumerated().map(|(idx, &dep_node)| (dep_node, idx)).collect(); + let index: FxHashMap<_, _> = nodes + .iter() + .enumerate() + .map(|(idx, &(dep_node, _))| (dep_node, SerializedDepNodeIndex::from_usize(idx))) + .collect(); - SerializedDepGraph { nodes, fingerprints, edge_list_indices, edge_list_data, index } + SerializedDepGraph { nodes: Some(nodes), index } } } -#[derive(Debug, Encodable, Decodable)] -pub struct NodeInfo { - node: DepNode, - fingerprint: Fingerprint, - edges: SmallVec<[DepNodeIndex; 8]>, -} - -struct Stat { - kind: K, - node_counter: u64, - edge_counter: u64, -} - struct EncoderState { encoder: FileEncoder, - total_node_count: usize, total_edge_count: usize, + nodes: IndexVec, u32)>, stats: Option>>, } @@ -174,22 +174,35 @@ impl EncoderState { Self { encoder, total_edge_count: 0, - total_node_count: 0, + nodes: IndexVec::default(), stats: record_stats.then(FxHashMap::default), } } + fn try_encode_node(&mut self, node: &NodeInfo) -> usize { + let encoder = &mut self.encoder; + let start_pos = encoder.write_mmap(&node.fingerprint); + let _pos = encoder.write_mmap::(&node.edges.len().try_into().unwrap()); + debug_assert_eq!(_pos, start_pos + std::mem::size_of::()); + let _pos = encoder.write_mmap_slice::(&node.edges[..]); + debug_assert_eq!(_pos, start_pos + std::mem::size_of::() + 4); + start_pos + } + fn encode_node( &mut self, node: &NodeInfo, record_graph: &Option>>, ) -> DepNodeIndex { - let index = DepNodeIndex::new(self.total_node_count); - self.total_node_count += 1; - let edge_count = node.edges.len(); self.total_edge_count += edge_count; + let position = self.try_encode_node(node); + debug_assert!(position & (std::mem::align_of::() - 1) == 0); + debug!(?position); + + let index = self.nodes.push((node.node, position.try_into().unwrap())); + if let Some(record_graph) = &record_graph { // Do not ICE when a query is called from within `with_query`. if let Some(record_graph) = &mut record_graph.try_lock() { @@ -205,21 +218,20 @@ impl EncoderState { stat.edge_counter += edge_count as u64; } - let encoder = &mut self.encoder; - node.encode(encoder); index } fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult { - let Self { mut encoder, total_node_count, total_edge_count, stats: _ } = self; + let Self { mut encoder, nodes, total_edge_count: _, stats: _ } = self; - let node_count = total_node_count.try_into().unwrap(); - let edge_count = total_edge_count.try_into().unwrap(); + let node_count = nodes.len().try_into().unwrap(); + let nodes_position = encoder.write_mmap_slice(&nodes.raw[..]); + let nodes_position = nodes_position.try_into().unwrap(); - debug!(?node_count, ?edge_count); + debug!(?node_count, ?nodes_position); debug!("position: {:?}", encoder.position()); IntEncodedWithFixedSize(node_count).encode(&mut encoder); - IntEncodedWithFixedSize(edge_count).encode(&mut encoder); + IntEncodedWithFixedSize(nodes_position).encode(&mut encoder); debug!("position: {:?}", encoder.position()); // Drop the encoder so that nothing is written after the counts. let result = encoder.finish(); @@ -232,12 +244,25 @@ impl EncoderState { } } +#[derive(Debug, Encodable, Decodable)] +pub struct NodeInfo { + node: DepNode, + fingerprint: Fingerprint, + edges: SmallVec<[DepNodeIndex; 8]>, +} + +struct Stat { + kind: K, + node_counter: u64, + edge_counter: u64, +} + pub struct GraphEncoder { status: Lock>, record_graph: Option>>, } -impl> GraphEncoder { +impl GraphEncoder { pub fn new( encoder: FileEncoder, prev_node_count: usize, @@ -270,11 +295,13 @@ impl> GraphEncoder { ----------------------------------------------\ ------------"; + let total_node_count = status.nodes.len(); + eprintln!("[incremental]"); eprintln!("[incremental] DepGraph Statistics"); eprintln!("{}", SEPARATOR); eprintln!("[incremental]"); - eprintln!("[incremental] Total Node Count: {}", status.total_node_count); + eprintln!("[incremental] Total Node Count: {}", total_node_count); eprintln!("[incremental] Total Edge Count: {}", status.total_edge_count); if cfg!(debug_assertions) { @@ -294,7 +321,7 @@ impl> GraphEncoder { for stat in stats { let node_kind_ratio = - (100.0 * (stat.node_counter as f64)) / (status.total_node_count as f64); + (100.0 * (stat.node_counter as f64)) / (total_node_count as f64); let node_kind_avg_edges = (stat.edge_counter as f64) / (stat.node_counter as f64); eprintln!( diff --git a/compiler/rustc_serialize/src/lib.rs b/compiler/rustc_serialize/src/lib.rs index e606f427335b7..ccdb13044f79a 100644 --- a/compiler/rustc_serialize/src/lib.rs +++ b/compiler/rustc_serialize/src/lib.rs @@ -9,12 +9,14 @@ Core encoding and decoding interfaces. html_playground_url = "https://play.rust-lang.org/", test(attr(allow(unused_variables), deny(warnings))) )] +#![feature(auto_traits)] #![feature(never_type)] #![feature(associated_type_bounds)] #![feature(min_specialization)] #![feature(core_intrinsics)] #![feature(maybe_uninit_slice)] #![feature(let_else)] +#![feature(negative_impls)] #![feature(new_uninit)] #![cfg_attr(test, feature(test))] #![allow(rustc::internal)] @@ -26,3 +28,10 @@ mod serialize; pub mod leb128; pub mod opaque; + +/// This trait is used to mark datastructures as safe for Mmap. +pub unsafe auto trait MmapSafe {} +impl<'a, T> !MmapSafe for &'a T {} +impl<'a, T> !MmapSafe for &'a mut T {} +impl !MmapSafe for *const T {} +impl !MmapSafe for *mut T {} diff --git a/compiler/rustc_serialize/src/opaque.rs b/compiler/rustc_serialize/src/opaque.rs index 5c17ef6ace2d5..16040e16a5798 100644 --- a/compiler/rustc_serialize/src/opaque.rs +++ b/compiler/rustc_serialize/src/opaque.rs @@ -1,5 +1,6 @@ use crate::leb128::{self, max_leb128_len}; use crate::serialize::{Decodable, Decoder, Encodable, Encoder}; +use crate::MmapSafe; use std::convert::TryInto; use std::fs::File; use std::io::{self, Write}; @@ -397,6 +398,38 @@ impl FileEncoder { let res = std::mem::replace(&mut self.res, Ok(())); res.map(|()| self.position()) } + + #[inline] + pub fn write_mmap(&mut self, val: &T) -> usize { + self.write_mmap_slice(std::slice::from_ref(val)) + } + + #[inline] + pub fn write_mmap_slice(&mut self, val: &[T]) -> usize { + const ALIGN: [u8; 64] = [0; 64]; + // First, we need to align the memory. + let pos = self.position(); + let align = std::mem::align_of::(); + let mask = align - 1; + let extra = pos & mask; + let padding = if extra == 0 { 0 } else { align - extra }; + self.write_all(&ALIGN[..padding]); + + let pos = pos + padding; + debug_assert!(pos & mask == 0); + debug_assert_eq!(pos, self.position()); + + // Now, we can write the data. + // SAFETY: The buffer has exactly the size for type `T`. + let raw_data: &[u8] = unsafe { + std::slice::from_raw_parts( + val.as_ptr() as *const u8, + val.len() * std::mem::size_of::(), + ) + }; + self.write_all(raw_data); + pos + } } impl Drop for FileEncoder { @@ -558,6 +591,27 @@ impl<'a> MemDecoder<'a> { pub fn advance(&mut self, bytes: usize) { self.position += bytes; } + + #[inline] + pub unsafe fn mmap_at(&self, pos: usize) -> &'a T { + let ret = self.mmap_slice_at::(pos, 1); + debug_assert_eq!(ret.len(), 1); + &ret[0] + } + + #[inline] + pub unsafe fn mmap_slice_at(&self, pos: usize, length: usize) -> &'a [T] { + let slice = &self.data[pos..]; + if cfg!(debug_assertions) { + // The decoder must ensure the position is properly aligned. + let _pre_padding = slice.as_ptr().align_offset(std::mem::align_of::()); + debug_assert_eq!(_pre_padding, 0); + + let byte_length = length * std::mem::size_of::(); + debug_assert!(byte_length <= slice.len()); + } + std::slice::from_raw_parts(slice.as_ptr() as *const T, length) + } } macro_rules! read_leb128 { From b136f26538726d2ff84018ea7938a9615a3e4af4 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Thu, 31 Mar 2022 23:52:56 +0200 Subject: [PATCH 2/5] Do not store the DepNode twice. --- Cargo.lock | 1 + compiler/rustc_query_system/Cargo.toml | 1 + .../src/dep_graph/serialized.rs | 34 +++++++++++++------ 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2325d0f3bf263..bc6d383538ccc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4371,6 +4371,7 @@ dependencies = [ name = "rustc_query_system" version = "0.0.0" dependencies = [ + "hashbrown", "parking_lot 0.11.2", "rustc-rayon-core", "rustc_arena", diff --git a/compiler/rustc_query_system/Cargo.toml b/compiler/rustc_query_system/Cargo.toml index b7787aeb8f765..441cc30de4ac6 100644 --- a/compiler/rustc_query_system/Cargo.toml +++ b/compiler/rustc_query_system/Cargo.toml @@ -8,6 +8,7 @@ doctest = false [dependencies] rustc_arena = { path = "../rustc_arena" } +hashbrown = { version = "0.12.0", features = ["raw"] } tracing = "0.1" rustc-rayon-core = { version = "0.4.0", optional = true } rustc_ast = { path = "../rustc_ast" } diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index ae0ec99bb35c2..038b33cc2dc88 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -14,8 +14,9 @@ use super::query::DepGraphQuery; use super::{DepKind, DepNode, DepNodeIndex}; +use hashbrown::raw::RawTable; use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxHasher}; use rustc_data_structures::memmap::Mmap; use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::profiling::SelfProfilerRef; @@ -25,6 +26,7 @@ use rustc_serialize::opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixed use rustc_serialize::{Decodable, Encodable}; use smallvec::SmallVec; use std::convert::TryInto; +use std::hash::{Hash, Hasher}; // The maximum value of `SerializedDepNodeIndex` leaves the upper two bits // unused so that we can store multiple index types in `CompressedHybridIndex`, @@ -69,7 +71,13 @@ pub struct SerializedDepGraph { /// The set of all DepNodes in the graph and their position in the mmap. nodes: Option, u32)]>>, /// Reciprocal map to `nodes`. - index: FxHashMap, SerializedDepNodeIndex>, + index: RawTable, +} + +fn hash_node(key: &DepNode) -> u64 { + let mut h = FxHasher::default(); + key.hash(&mut h); + h.finish() } impl Default for SerializedDepGraph { @@ -91,7 +99,10 @@ impl SerializedDepGraph { #[inline] pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { - self.index.get(dep_node).cloned() + let nodes = self.nodes.as_ref()?; + let hash = hash_node(dep_node); + let index = self.index.get(hash, |&i| *dep_node == nodes[i as usize].0)?; + Some(SerializedDepNodeIndex::from_u32(*index)) } #[inline] @@ -112,7 +123,7 @@ impl SerializedDepGraph { #[inline] pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option { - let index = self.index.get(dep_node).cloned()?; + let index = self.node_to_index_opt(dep_node)?; Some(self.fingerprint_by_index(index)) } @@ -131,7 +142,7 @@ impl SerializedDepGraph { } pub fn node_count(&self) -> usize { - self.index.len() + if let Some(ref nodes) = self.nodes { nodes.len() } else { 0 } } #[instrument(level = "debug", skip(mmap))] @@ -152,11 +163,14 @@ impl SerializedDepGraph { unsafe { d.mmap_slice_at::<(DepNode, u32)>(nodes_position, node_count) } }); - let index: FxHashMap<_, _> = nodes - .iter() - .enumerate() - .map(|(idx, &(dep_node, _))| (dep_node, SerializedDepNodeIndex::from_usize(idx))) - .collect(); + let mut index = RawTable::with_capacity(node_count); + let mut known_hashes = Vec::with_capacity(node_count); + for (idx, (dep_node, _)) in nodes.iter().enumerate() { + let idx = idx.try_into().unwrap(); + let hash = hash_node(dep_node); + known_hashes.push(hash); + index.insert(hash, idx, |&i| known_hashes[i as usize]); + } SerializedDepGraph { nodes: Some(nodes), index } } From 8b9e38c557f3bd3beb349044ce12f2ac8abeebc8 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 1 May 2022 14:14:12 +0200 Subject: [PATCH 3/5] Remove useless bounds. --- compiler/rustc_query_system/src/dep_graph/dep_node.rs | 2 +- compiler/rustc_query_system/src/dep_graph/serialized.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs index 162c274d8a296..77d2b84dbdee1 100644 --- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs +++ b/compiler/rustc_query_system/src/dep_graph/dep_node.rs @@ -50,7 +50,7 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use std::fmt; use std::hash::Hash; -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct DepNode { pub kind: K, pub hash: PackedFingerprint, diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index 038b33cc2dc88..2bbd46cfb6cef 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -258,7 +258,7 @@ impl EncoderState { } } -#[derive(Debug, Encodable, Decodable)] +#[derive(Debug)] pub struct NodeInfo { node: DepNode, fingerprint: Fingerprint, From 620d16ab9d86477ddcbdd03c9e380786af8ddef4 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 2 Jul 2022 15:20:48 +0200 Subject: [PATCH 4/5] Go back to LEB128 for edges. --- .../rustc_query_system/src/dep_graph/graph.rs | 3 +-- .../src/dep_graph/serialized.rs | 18 +++++++----------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 8ff56132749df..26ff6f005f7c2 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -731,8 +731,7 @@ impl DepGraph { debug_assert_eq!(data.previous.index_to_node(prev_dep_node_index), *dep_node); let prev_deps = data.previous.edge_targets_from(prev_dep_node_index); - - for &dep_dep_node_index in prev_deps { + for dep_dep_node_index in prev_deps { self.try_mark_parent_green(tcx, data, dep_dep_node_index, dep_node)? } diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index 2bbd46cfb6cef..bb51b3e932837 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -128,16 +128,13 @@ impl SerializedDepGraph { } #[inline] - pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] { + pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> Vec { // The encoder has checked that there is no padding there. - if let Some(decoder) = self.decoder_at(source) { - let position = std::mem::size_of::(); - let &length = unsafe { decoder.mmap_at::(position) }; - unsafe { - decoder.mmap_slice_at::(position + 4, length as usize) - } + if let Some(ref mut decoder) = self.decoder_at(source) { + decoder.set_position(std::mem::size_of::()); + Decodable::decode(decoder) } else { - &[] + Vec::new() } } @@ -196,10 +193,9 @@ impl EncoderState { fn try_encode_node(&mut self, node: &NodeInfo) -> usize { let encoder = &mut self.encoder; let start_pos = encoder.write_mmap(&node.fingerprint); - let _pos = encoder.write_mmap::(&node.edges.len().try_into().unwrap()); + let _pos = encoder.position(); debug_assert_eq!(_pos, start_pos + std::mem::size_of::()); - let _pos = encoder.write_mmap_slice::(&node.edges[..]); - debug_assert_eq!(_pos, start_pos + std::mem::size_of::() + 4); + node.edges.encode(encoder); start_pos } From 9b4c844842014b7cd9a4cc43b956e432119fa327 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 29 Jul 2022 19:02:07 +0200 Subject: [PATCH 5/5] Decode edges using an iterator. --- .../rustc_query_system/src/dep_graph/graph.rs | 3 +-- .../src/dep_graph/serialized.rs | 16 +++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 26ff6f005f7c2..1e26ca6eb27a7 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -1180,8 +1180,7 @@ impl CurrentDepGraph { prev_graph.fingerprint_by_index(prev_index), prev_graph .edge_targets_from(prev_index) - .iter() - .map(|i| prev_index_to_index[*i].unwrap()) + .map(|i| prev_index_to_index[i].unwrap()) .collect(), ); prev_index_to_index[prev_index] = Some(dep_node_index); diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index bb51b3e932837..c120d144b426d 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -23,7 +23,7 @@ use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sync::Lock; use rustc_index::vec::IndexVec; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder}; -use rustc_serialize::{Decodable, Encodable}; +use rustc_serialize::{Decodable, Decoder, Encodable}; use smallvec::SmallVec; use std::convert::TryInto; use std::hash::{Hash, Hasher}; @@ -128,14 +128,20 @@ impl SerializedDepGraph { } #[inline] - pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> Vec { + pub fn edge_targets_from( + &self, + source: SerializedDepNodeIndex, + ) -> impl Iterator + '_ { // The encoder has checked that there is no padding there. - if let Some(ref mut decoder) = self.decoder_at(source) { + if let Some(mut decoder) = self.decoder_at(source) { decoder.set_position(std::mem::size_of::()); - Decodable::decode(decoder) + let len = decoder.read_usize(); + Some((0..len).map(move |_| SerializedDepNodeIndex::decode(&mut decoder))) } else { - Vec::new() + None } + .into_iter() + .flatten() } pub fn node_count(&self) -> usize {