diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index de7b8c63fc87a..bebd9723740f8 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -4,6 +4,7 @@ use either::Right; use rustc_const_eval::const_eval::CheckAlignment; +use rustc_data_structures::fx::FxHashSet; use rustc_hir::def::DefKind; use rustc_index::bit_set::BitSet; use rustc_index::vec::IndexVec; @@ -151,12 +152,17 @@ impl<'tcx> MirPass<'tcx> for ConstProp { pub struct ConstPropMachine<'mir, 'tcx> { /// The virtual call stack. stack: Vec>, + pub written_only_inside_own_block_locals: FxHashSet, pub can_const_prop: IndexVec, } impl ConstPropMachine<'_, '_> { pub fn new(can_const_prop: IndexVec) -> Self { - Self { stack: Vec::new(), can_const_prop } + Self { + stack: Vec::new(), + written_only_inside_own_block_locals: Default::default(), + can_const_prop, + } } } @@ -249,7 +255,10 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> "tried to write to a local that is marked as not propagatable" ) } - ConstPropMode::OnlyInsideOwnBlock | ConstPropMode::FullConstProp => {} + ConstPropMode::OnlyInsideOwnBlock => { + ecx.machine.written_only_inside_own_block_locals.insert(local); + } + ConstPropMode::FullConstProp => {} } ecx.machine.stack[frame].locals[local].access_mut() } @@ -416,6 +425,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { ecx.frame_mut().locals[local].value = LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); + ecx.machine.written_only_inside_own_block_locals.remove(&local); } /// Returns the value, if any, of evaluating `c`. @@ -693,7 +703,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } } - fn ensure_not_propagated(&mut self, local: Local) { + fn ensure_not_propagated(&self, local: Local) { if cfg!(debug_assertions) { assert!( self.get_const(local.into()).is_none() @@ -963,17 +973,31 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { // We remove all Locals which are restricted in propagation to their containing blocks and // which were modified in the current block. // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`. - let can_const_prop = std::mem::take(&mut self.ecx.machine.can_const_prop); - for (local, &mode) in can_const_prop.iter_enumerated() { - match mode { - ConstPropMode::FullConstProp => {} - ConstPropMode::NoPropagation => self.ensure_not_propagated(local), - ConstPropMode::OnlyInsideOwnBlock => { - Self::remove_const(&mut self.ecx, local); - self.ensure_not_propagated(local); + let mut written_only_inside_own_block_locals = + std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals); + + // This loop can get very hot for some bodies: it check each local in each bb. + // To avoid this quadratic behaviour, we only clear the locals that were modified inside + // the current block. + for local in written_only_inside_own_block_locals.drain() { + debug_assert_eq!( + self.ecx.machine.can_const_prop[local], + ConstPropMode::OnlyInsideOwnBlock + ); + Self::remove_const(&mut self.ecx, local); + } + self.ecx.machine.written_only_inside_own_block_locals = + written_only_inside_own_block_locals; + + if cfg!(debug_assertions) { + for (local, &mode) in self.ecx.machine.can_const_prop.iter_enumerated() { + match mode { + ConstPropMode::FullConstProp => {} + ConstPropMode::NoPropagation | ConstPropMode::OnlyInsideOwnBlock => { + self.ensure_not_propagated(local); + } } } } - self.ecx.machine.can_const_prop = can_const_prop; } } diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index 68e50070e56e1..45bd98f39d2ba 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -247,6 +247,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { ecx.frame_mut().locals[local].value = LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); + ecx.machine.written_only_inside_own_block_locals.remove(&local); } fn lint_root(&self, source_info: SourceInfo) -> Option { @@ -484,7 +485,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { Some(()) } - fn ensure_not_propagated(&mut self, local: Local) { + fn ensure_not_propagated(&self, local: Local) { if cfg!(debug_assertions) { assert!( self.get_const(local.into()).is_none() @@ -691,17 +692,31 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { // We remove all Locals which are restricted in propagation to their containing blocks and // which were modified in the current block. // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`. - let can_const_prop = std::mem::take(&mut self.ecx.machine.can_const_prop); - for (local, &mode) in can_const_prop.iter_enumerated() { - match mode { - ConstPropMode::FullConstProp => {} - ConstPropMode::NoPropagation => self.ensure_not_propagated(local), - ConstPropMode::OnlyInsideOwnBlock => { - Self::remove_const(&mut self.ecx, local); - self.ensure_not_propagated(local); + let mut written_only_inside_own_block_locals = + std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals); + + // This loop can get very hot for some bodies: it check each local in each bb. + // To avoid this quadratic behaviour, we only clear the locals that were modified inside + // the current block. + for local in written_only_inside_own_block_locals.drain() { + debug_assert_eq!( + self.ecx.machine.can_const_prop[local], + ConstPropMode::OnlyInsideOwnBlock + ); + Self::remove_const(&mut self.ecx, local); + } + self.ecx.machine.written_only_inside_own_block_locals = + written_only_inside_own_block_locals; + + if cfg!(debug_assertions) { + for (local, &mode) in self.ecx.machine.can_const_prop.iter_enumerated() { + match mode { + ConstPropMode::FullConstProp => {} + ConstPropMode::NoPropagation | ConstPropMode::OnlyInsideOwnBlock => { + self.ensure_not_propagated(local); + } } } } - self.ecx.machine.can_const_prop = can_const_prop; } }