diff --git a/Cargo.lock b/Cargo.lock index 49b89c71994a8..1d40b8acc666e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1365,9 +1365,9 @@ dependencies = [ [[package]] name = "ena" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e5d13ca2353ab7d0230988629def93914a8c4015f621f9b13ed2955614731d" +checksum = "c533630cf40e9caa44bd91aadc88a75d75a4c3a12b4cfde353cbed41daa1e1f1" dependencies = [ "log", ] diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index 29cb2c0a33e6c..f2a904a6654da 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" arrayvec = { version = "0.7", default-features = false } bitflags = "1.2.1" cfg-if = "1.0" -ena = "0.14.1" +ena = "0.14.2" indexmap = { version = "1.9.1" } jobserver_crate = { version = "0.1.13", package = "jobserver" } libc = "0.2" diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index 91abdaadabdbd..27a869eb7cdb0 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -97,7 +97,17 @@ pub trait ObligationProcessor { type Error: Debug; type OUT: OutcomeTrait>; - fn needs_process_obligation(&self, obligation: &Self::Obligation) -> bool; + /// Implementations can provide a fast-path to obligation-processing + /// by counting the prefix of the passed iterator for which + /// `needs_process_obligation` would return false. + fn skippable_obligations<'a>( + &'a self, + _it: impl Iterator, + ) -> usize { + 0 + } + + fn needs_process_obligation(&self, _obligation: &Self::Obligation) -> bool; fn process_obligation( &mut self, @@ -416,6 +426,10 @@ impl ObligationForest { loop { let mut has_changed = false; + // This is the super fast path for cheap-to-check conditions. + let mut index = + processor.skippable_obligations(self.nodes.iter().map(|n| &n.obligation)); + // Note that the loop body can append new nodes, and those new nodes // will then be processed by subsequent iterations of the loop. // @@ -424,9 +438,8 @@ impl ObligationForest { // `for index in 0..self.nodes.len() { ... }` because the range would // be computed with the initial length, and we would miss the appended // nodes. Therefore we use a `while` loop. - let mut index = 0; while let Some(node) = self.nodes.get_mut(index) { - // This test is extremely hot. + // This is the moderately fast path when the prefix skipping above didn't work out. if node.state.get() != NodeState::Pending || !processor.needs_process_obligation(&node.obligation) { diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index 96e7c095d341c..aeb4ddb421259 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -187,6 +187,16 @@ impl<'tcx> InferCtxtInner<'tcx> { self.projection_cache.with_log(&mut self.undo_log) } + #[inline] + fn try_type_variables_probe_ref( + &self, + vid: ty::TyVid, + ) -> Option<&type_variable::TypeVariableValue<'tcx>> { + // Uses a read-only view of the unification table, this way we don't + // need an undo log. + self.type_variable_storage.eq_relations_ref().try_probe_value(vid) + } + #[inline] fn type_variables(&mut self) -> type_variable::TypeVariableTable<'_, 'tcx> { self.type_variable_storage.with_log(&mut self.undo_log) @@ -1646,6 +1656,28 @@ impl<'tcx> InferCtxt<'tcx> { tcx.const_eval_resolve_for_typeck(param_env_erased, unevaluated, span) } + /// The returned function is used in a fast path. If it returns `true` the variable is + /// unchanged, `false` indicates that the status is unknown. + #[inline] + pub fn is_ty_infer_var_definitely_unchanged<'a>( + &'a self, + ) -> (impl Fn(TyOrConstInferVar<'tcx>) -> bool + 'a) { + // This hoists the borrow/release out of the loop body. + let inner = self.inner.try_borrow(); + + return move |infer_var: TyOrConstInferVar<'tcx>| match (infer_var, &inner) { + (TyOrConstInferVar::Ty(ty_var), Ok(inner)) => { + use self::type_variable::TypeVariableValue; + + match inner.try_type_variables_probe_ref(ty_var) { + Some(TypeVariableValue::Unknown { .. }) => true, + _ => false, + } + } + _ => false, + }; + } + /// `ty_or_const_infer_var_changed` is equivalent to one of these two: /// * `shallow_resolve(ty) != ty` (where `ty.kind = ty::Infer(_)`) /// * `shallow_resolve(ct) != ct` (where `ct.kind = ty::ConstKind::Infer(_)`) diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs index 263c6a47dd2af..f7ab05b2d498a 100644 --- a/compiler/rustc_infer/src/infer/type_variable.rs +++ b/compiler/rustc_infer/src/infer/type_variable.rs @@ -190,6 +190,11 @@ impl<'tcx> TypeVariableStorage<'tcx> { ) -> TypeVariableTable<'a, 'tcx> { TypeVariableTable { storage: self, undo_log } } + + #[inline] + pub(crate) fn eq_relations_ref(&self) -> &ut::UnificationTableStorage> { + &self.eq_relations + } } impl<'tcx> TypeVariableTable<'_, 'tcx> { diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index 1aaadae12ddd9..23754480fcfb0 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -211,6 +211,29 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> { type Error = FulfillmentErrorCode<'tcx>; type OUT = Outcome; + /// Compared to `needs_process_obligation` this and its callees + /// contain some optimizations that come at the price of false negatives. + /// + /// They + /// - reduce branching by covering only the most common case + /// - take a read-only view of the unification tables which allows skipping undo_log + /// construction. + /// - bail out on value-cache misses in ena to avoid pointer chasing + /// - hoist RefCell locking out of the loop + #[inline] + fn skippable_obligations<'b>( + &'b self, + it: impl Iterator, + ) -> usize { + let is_unchanged = self.selcx.infcx.is_ty_infer_var_definitely_unchanged(); + + it.take_while(|o| match o.stalled_on.as_slice() { + [o] => is_unchanged(*o), + _ => false, + }) + .count() + } + /// Identifies whether a predicate obligation needs processing. /// /// This is always inlined because it has a single callsite and it is