diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index ef0c763ac2038..0fc1da27b8f28 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -1189,7 +1189,7 @@ impl FieldsShape { } FieldsShape::Array { stride, count } => { let i = u64::try_from(i).unwrap(); - assert!(i < count); + assert!(i < count, "tried to access field {} of array with {} fields", i, count); stride * i } FieldsShape::Arbitrary { ref offsets, .. } => offsets[FieldIdx::from_usize(i)], diff --git a/compiler/rustc_const_eval/messages.ftl b/compiler/rustc_const_eval/messages.ftl index d8eade5bd2a0e..671c2be1de9d2 100644 --- a/compiler/rustc_const_eval/messages.ftl +++ b/compiler/rustc_const_eval/messages.ftl @@ -408,8 +408,11 @@ const_eval_undefined_behavior = const_eval_undefined_behavior_note = The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. +const_eval_uninhabited_enum_tag = {$front_matter}: encountered an uninhabited enum variant +const_eval_uninhabited_enum_variant_read = + read discriminant of an uninhabited enum variant const_eval_uninhabited_enum_variant_written = - writing discriminant of an uninhabited enum + writing discriminant of an uninhabited enum variant const_eval_uninhabited_val = {$front_matter}: encountered a value of uninhabited type `{$ty}` const_eval_uninit = {$front_matter}: encountered uninitialized bytes const_eval_uninit_bool = {$front_matter}: encountered uninitialized memory, but expected a boolean @@ -423,8 +426,6 @@ const_eval_uninit_int = {$front_matter}: encountered uninitialized memory, but e const_eval_uninit_raw_ptr = {$front_matter}: encountered uninitialized memory, but expected a raw pointer const_eval_uninit_ref = {$front_matter}: encountered uninitialized memory, but expected a reference const_eval_uninit_str = {$front_matter}: encountered uninitialized data in `str` -const_eval_uninit_unsized_local = - unsized local is used while uninitialized const_eval_unreachable = entering unreachable code const_eval_unreachable_unwind = unwinding past a stack frame that does not allow unwinding diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs index 0a3c3914ff971..ef31155215a1b 100644 --- a/compiler/rustc_const_eval/src/const_eval/mod.rs +++ b/compiler/rustc_const_eval/src/const_eval/mod.rs @@ -101,8 +101,8 @@ pub(crate) fn try_destructure_mir_constant_for_diagnostics<'tcx>( return None; } ty::Adt(def, _) => { - let variant = ecx.read_discriminant(&op).ok()?.1; - let down = ecx.operand_downcast(&op, variant).ok()?; + let variant = ecx.read_discriminant(&op).ok()?; + let down = ecx.project_downcast(&op, variant).ok()?; (def.variants()[variant].fields.len(), Some(variant), down) } ty::Tuple(args) => (args.len(), None, op), @@ -111,7 +111,7 @@ pub(crate) fn try_destructure_mir_constant_for_diagnostics<'tcx>( let fields_iter = (0..field_count) .map(|i| { - let field_op = ecx.operand_field(&down, i).ok()?; + let field_op = ecx.project_field(&down, i).ok()?; let val = op_to_const(&ecx, &field_op); Some((val, field_op.layout.ty)) }) diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs index 9531d973eb323..b519bcdf4a339 100644 --- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs +++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs @@ -2,11 +2,11 @@ use super::eval_queries::{mk_eval_cx, op_to_const}; use super::machine::CompileTimeEvalContext; use super::{ValTreeCreationError, ValTreeCreationResult, VALTREE_MAX_NODES}; use crate::const_eval::CanAccessStatics; +use crate::interpret::MPlaceTy; use crate::interpret::{ intern_const_alloc_recursive, ConstValue, ImmTy, Immediate, InternKind, MemPlaceMeta, - MemoryKind, PlaceTy, Scalar, + MemoryKind, PlaceTy, Projectable, Scalar, }; -use crate::interpret::{MPlaceTy, Value}; use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt}; use rustc_span::source_map::DUMMY_SP; use rustc_target::abi::{Align, FieldIdx, VariantIdx, FIRST_VARIANT}; @@ -20,7 +20,7 @@ fn branches<'tcx>( num_nodes: &mut usize, ) -> ValTreeCreationResult<'tcx> { let place = match variant { - Some(variant) => ecx.mplace_downcast(&place, variant).unwrap(), + Some(variant) => ecx.project_downcast(place, variant).unwrap(), None => *place, }; let variant = variant.map(|variant| Some(ty::ValTree::Leaf(ScalarInt::from(variant.as_u32())))); @@ -28,7 +28,7 @@ fn branches<'tcx>( let mut fields = Vec::with_capacity(n); for i in 0..n { - let field = ecx.mplace_field(&place, i).unwrap(); + let field = ecx.project_field(&place, i).unwrap(); let valtree = const_to_valtree_inner(ecx, &field, num_nodes)?; fields.push(Some(valtree)); } @@ -55,13 +55,11 @@ fn slice_branches<'tcx>( place: &MPlaceTy<'tcx>, num_nodes: &mut usize, ) -> ValTreeCreationResult<'tcx> { - let n = place - .len(&ecx.tcx.tcx) - .unwrap_or_else(|_| panic!("expected to use len of place {:?}", place)); + let n = place.len(ecx).unwrap_or_else(|_| panic!("expected to use len of place {:?}", place)); let mut elems = Vec::with_capacity(n as usize); for i in 0..n { - let place_elem = ecx.mplace_index(place, i).unwrap(); + let place_elem = ecx.project_index(place, i).unwrap(); let valtree = const_to_valtree_inner(ecx, &place_elem, num_nodes)?; elems.push(valtree); } @@ -132,7 +130,7 @@ pub(crate) fn const_to_valtree_inner<'tcx>( bug!("uninhabited types should have errored and never gotten converted to valtree") } - let Ok((_, variant)) = ecx.read_discriminant(&place.into()) else { + let Ok(variant) = ecx.read_discriminant(&place.into()) else { return Err(ValTreeCreationError::Other); }; branches(ecx, place, def.variant(variant).fields.len(), def.is_enum().then_some(variant), num_nodes) @@ -386,7 +384,7 @@ fn valtree_into_mplace<'tcx>( debug!(?variant); ( - place.project_downcast(ecx, variant_idx).unwrap(), + ecx.project_downcast(place, variant_idx).unwrap(), &branches[1..], Some(variant_idx), ) @@ -401,7 +399,7 @@ fn valtree_into_mplace<'tcx>( debug!(?i, ?inner_valtree); let mut place_inner = match ty.kind() { - ty::Str | ty::Slice(_) => ecx.mplace_index(&place, i as u64).unwrap(), + ty::Str | ty::Slice(_) => ecx.project_index(place, i as u64).unwrap(), _ if !ty.is_sized(*ecx.tcx, ty::ParamEnv::empty()) && i == branches.len() - 1 => { @@ -441,7 +439,7 @@ fn valtree_into_mplace<'tcx>( ) .unwrap() } - _ => ecx.mplace_field(&place_adjusted, i).unwrap(), + _ => ecx.project_field(&place_adjusted, i).unwrap(), }; debug!(?place_inner); diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs index ca38cce710e60..e1109e584b7bf 100644 --- a/compiler/rustc_const_eval/src/errors.rs +++ b/compiler/rustc_const_eval/src/errors.rs @@ -511,7 +511,8 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> { InvalidUninitBytes(Some(_)) => const_eval_invalid_uninit_bytes, DeadLocal => const_eval_dead_local, ScalarSizeMismatch(_) => const_eval_scalar_size_mismatch, - UninhabitedEnumVariantWritten => const_eval_uninhabited_enum_variant_written, + UninhabitedEnumVariantWritten(_) => const_eval_uninhabited_enum_variant_written, + UninhabitedEnumVariantRead(_) => const_eval_uninhabited_enum_variant_read, Validation(e) => e.diagnostic_message(), Custom(x) => (x.msg)(), } @@ -535,7 +536,8 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> { | InvalidMeta(InvalidMetaKind::TooBig) | InvalidUninitBytes(None) | DeadLocal - | UninhabitedEnumVariantWritten => {} + | UninhabitedEnumVariantWritten(_) + | UninhabitedEnumVariantRead(_) => {} BoundsCheckFailed { len, index } => { builder.set_arg("len", len); builder.set_arg("index", index); @@ -623,6 +625,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> { UnsafeCell => const_eval_unsafe_cell, UninhabitedVal { .. } => const_eval_uninhabited_val, InvalidEnumTag { .. } => const_eval_invalid_enum_tag, + UninhabitedEnumTag => const_eval_uninhabited_enum_tag, UninitEnumTag => const_eval_uninit_enum_tag, UninitStr => const_eval_uninit_str, Uninit { expected: ExpectedKind::Bool } => const_eval_uninit_bool, @@ -760,7 +763,8 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> { | InvalidMetaSliceTooLarge { .. } | InvalidMetaTooLarge { .. } | DanglingPtrUseAfterFree { .. } - | DanglingPtrOutOfBounds { .. } => {} + | DanglingPtrOutOfBounds { .. } + | UninhabitedEnumTag => {} } } } @@ -835,7 +839,9 @@ impl<'tcx> ReportErrorExt for InvalidProgramInfo<'tcx> { rustc_middle::error::middle_adjust_for_foreign_abi_error } InvalidProgramInfo::SizeOfUnsizedType(_) => const_eval_size_of_unsized, - InvalidProgramInfo::UninitUnsizedLocal => const_eval_uninit_unsized_local, + InvalidProgramInfo::ConstPropNonsense => { + panic!("We had const-prop nonsense, this should never be printed") + } } } fn add_args( @@ -846,7 +852,7 @@ impl<'tcx> ReportErrorExt for InvalidProgramInfo<'tcx> { match self { InvalidProgramInfo::TooGeneric | InvalidProgramInfo::AlreadyReported(_) - | InvalidProgramInfo::UninitUnsizedLocal => {} + | InvalidProgramInfo::ConstPropNonsense => {} InvalidProgramInfo::Layout(e) => { let diag: DiagnosticBuilder<'_, ()> = e.into_diagnostic().into_diagnostic(handler); for (name, val) in diag.args() { diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs index dd7a1fcc16594..977e49b63431a 100644 --- a/compiler/rustc_const_eval/src/interpret/cast.rs +++ b/compiler/rustc_const_eval/src/interpret/cast.rs @@ -420,8 +420,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { if cast_ty_field.is_zst() { continue; } - let src_field = self.operand_field(src, i)?; - let dst_field = self.place_field(dest, i)?; + let src_field = self.project_field(src, i)?; + let dst_field = self.project_field(dest, i)?; if src_field.layout.ty == cast_ty_field.ty { self.copy_op(&src_field, &dst_field, /*allow_transmute*/ false)?; } else { diff --git a/compiler/rustc_const_eval/src/interpret/discriminant.rs b/compiler/rustc_const_eval/src/interpret/discriminant.rs index f23a455c2ca30..aff86d5f48668 100644 --- a/compiler/rustc_const_eval/src/interpret/discriminant.rs +++ b/compiler/rustc_const_eval/src/interpret/discriminant.rs @@ -1,6 +1,6 @@ //! Functions for reading and writing discriminants of multi-variant layouts (enums and generators). -use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt}; +use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt, TyAndLayout}; use rustc_middle::{mir, ty}; use rustc_target::abi::{self, TagEncoding}; use rustc_target::abi::{VariantIdx, Variants}; @@ -22,7 +22,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // When evaluating we will always error before even getting here, but ConstProp 'executes' // dead code, so we cannot ICE here. if dest.layout.for_variant(self, variant_index).abi.is_uninhabited() { - throw_ub!(UninhabitedEnumVariantWritten) + throw_ub!(UninhabitedEnumVariantWritten(variant_index)) } match dest.layout.variants { @@ -47,7 +47,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let size = tag_layout.size(self); let tag_val = size.truncate(discr_val); - let tag_dest = self.place_field(dest, tag_field)?; + let tag_dest = self.project_field(dest, tag_field)?; self.write_scalar(Scalar::from_uint(tag_val, size), &tag_dest)?; } abi::Variants::Multiple { @@ -78,7 +78,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { &niche_start_val, )?; // Write result. - let niche_dest = self.place_field(dest, tag_field)?; + let niche_dest = self.project_field(dest, tag_field)?; self.write_immediate(*tag_val, &niche_dest)?; } } @@ -93,7 +93,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { pub fn read_discriminant( &self, op: &OpTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, (Scalar, VariantIdx)> { + ) -> InterpResult<'tcx, VariantIdx> { trace!("read_discriminant_value {:#?}", op.layout); // Get type and layout of the discriminant. let discr_layout = self.layout_of(op.layout.ty.discriminant_ty(*self.tcx))?; @@ -106,19 +106,22 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // straight-forward (`TagEncoding::Direct`) or with a niche (`TagEncoding::Niche`). let (tag_scalar_layout, tag_encoding, tag_field) = match op.layout.variants { Variants::Single { index } => { - let discr = match op.layout.ty.discriminant_for_variant(*self.tcx, index) { - Some(discr) => { - // This type actually has discriminants. - assert_eq!(discr.ty, discr_layout.ty); - Scalar::from_uint(discr.val, discr_layout.size) + // Do some extra checks on enums. + if op.layout.ty.is_enum() { + // Hilariously, `Single` is used even for 0-variant enums. + // (See https://github.com/rust-lang/rust/issues/89765). + if matches!(op.layout.ty.kind(), ty::Adt(def, ..) if def.variants().is_empty()) + { + throw_ub!(UninhabitedEnumVariantRead(index)) } - None => { - // On a type without actual discriminants, variant is 0. - assert_eq!(index.as_u32(), 0); - Scalar::from_uint(index.as_u32(), discr_layout.size) + // For consisteny with `write_discriminant`, and to make sure that + // `project_downcast` cannot fail due to strange layouts, we declare immediate UB + // for uninhabited variants. + if op.layout.for_variant(self, index).abi.is_uninhabited() { + throw_ub!(UninhabitedEnumVariantRead(index)) } - }; - return Ok((discr, index)); + } + return Ok(index); } Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => { (tag, tag_encoding, tag_field) @@ -138,13 +141,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let tag_layout = self.layout_of(tag_scalar_layout.primitive().to_int_ty(*self.tcx))?; // Read tag and sanity-check `tag_layout`. - let tag_val = self.read_immediate(&self.operand_field(op, tag_field)?)?; + let tag_val = self.read_immediate(&self.project_field(op, tag_field)?)?; assert_eq!(tag_layout.size, tag_val.layout.size); assert_eq!(tag_layout.abi.is_signed(), tag_val.layout.abi.is_signed()); trace!("tag value: {}", tag_val); // Figure out which discriminant and variant this corresponds to. - Ok(match *tag_encoding { + let index = match *tag_encoding { TagEncoding::Direct => { let scalar = tag_val.to_scalar(); // Generate a specific error if `tag_val` is not an integer. @@ -172,7 +175,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } .ok_or_else(|| err_ub!(InvalidTag(Scalar::from_uint(tag_bits, tag_layout.size))))?; // Return the cast value, and the index. - (discr_val, index.0) + index.0 } TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => { let tag_val = tag_val.to_scalar(); @@ -230,7 +233,32 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Compute the size of the scalar we need to return. // No need to cast, because the variant index directly serves as discriminant and is // encoded in the tag. - (Scalar::from_uint(variant.as_u32(), discr_layout.size), variant) + variant + } + }; + // For consisteny with `write_discriminant`, and to make sure that `project_downcast` cannot fail due to strange layouts, we declare immediate UB for uninhabited variants. + if op.layout.for_variant(self, index).abi.is_uninhabited() { + throw_ub!(UninhabitedEnumVariantRead(index)) + } + Ok(index) + } + + pub fn discriminant_for_variant( + &self, + layout: TyAndLayout<'tcx>, + variant: VariantIdx, + ) -> InterpResult<'tcx, Scalar> { + let discr_layout = self.layout_of(layout.ty.discriminant_ty(*self.tcx))?; + Ok(match layout.ty.discriminant_for_variant(*self.tcx, variant) { + Some(discr) => { + // This type actually has discriminants. + assert_eq!(discr.ty, discr_layout.ty); + Scalar::from_uint(discr.val, discr_layout.size) + } + None => { + // On a type without actual discriminants, variant is 0. + assert_eq!(variant.as_u32(), 0); + Scalar::from_uint(variant.as_u32(), discr_layout.size) } }) } diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index 04e046fbda352..0e6125388a603 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -1014,9 +1014,12 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> std::fmt::Debug { fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self.place { - Place::Local { frame, local } => { + Place::Local { frame, local, offset } => { let mut allocs = Vec::new(); write!(fmt, "{:?}", local)?; + if let Some(offset) = offset { + write!(fmt, "+{:#x}", offset.bytes())?; + } if frame != self.ecx.frame_idx() { write!(fmt, " ({} frames up)", self.ecx.frame_idx() - frame)?; } diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs index 107e5bec6141b..3a7fe8bd4781b 100644 --- a/compiler/rustc_const_eval/src/interpret/intern.rs +++ b/compiler/rustc_const_eval/src/interpret/intern.rs @@ -164,75 +164,6 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory &self.ecx } - fn visit_aggregate( - &mut self, - mplace: &MPlaceTy<'tcx>, - fields: impl Iterator>, - ) -> InterpResult<'tcx> { - // We want to walk the aggregate to look for references to intern. While doing that we - // also need to take special care of interior mutability. - // - // As an optimization, however, if the allocation does not contain any references: we don't - // need to do the walk. It can be costly for big arrays for example (e.g. issue #93215). - let is_walk_needed = |mplace: &MPlaceTy<'tcx>| -> InterpResult<'tcx, bool> { - // ZSTs cannot contain pointers, we can avoid the interning walk. - if mplace.layout.is_zst() { - return Ok(false); - } - - // Now, check whether this allocation could contain references. - // - // Note, this check may sometimes not be cheap, so we only do it when the walk we'd like - // to avoid could be expensive: on the potentially larger types, arrays and slices, - // rather than on all aggregates unconditionally. - if matches!(mplace.layout.ty.kind(), ty::Array(..) | ty::Slice(..)) { - let Some((size, align)) = self.ecx.size_and_align_of_mplace(&mplace)? else { - // We do the walk if we can't determine the size of the mplace: we may be - // dealing with extern types here in the future. - return Ok(true); - }; - - // If there is no provenance in this allocation, it does not contain references - // that point to another allocation, and we can avoid the interning walk. - if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, align)? { - if !alloc.has_provenance() { - return Ok(false); - } - } else { - // We're encountering a ZST here, and can avoid the walk as well. - return Ok(false); - } - } - - // In the general case, we do the walk. - Ok(true) - }; - - // If this allocation contains no references to intern, we avoid the potentially costly - // walk. - // - // We can do this before the checks for interior mutability below, because only references - // are relevant in that situation, and we're checking if there are any here. - if !is_walk_needed(mplace)? { - return Ok(()); - } - - if let Some(def) = mplace.layout.ty.ty_adt_def() { - if def.is_unsafe_cell() { - // We are crossing over an `UnsafeCell`, we can mutate again. This means that - // References we encounter inside here are interned as pointing to mutable - // allocations. - // Remember the `old` value to handle nested `UnsafeCell`. - let old = std::mem::replace(&mut self.inside_unsafe_cell, true); - let walked = self.walk_aggregate(mplace, fields); - self.inside_unsafe_cell = old; - return walked; - } - } - - self.walk_aggregate(mplace, fields) - } - fn visit_value(&mut self, mplace: &MPlaceTy<'tcx>) -> InterpResult<'tcx> { // Handle Reference types, as these are the only types with provenance supported by const eval. // Raw pointers (and boxes) are handled by the `leftover_allocations` logic. @@ -315,7 +246,63 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory } Ok(()) } else { - // Not a reference -- proceed recursively. + // Not a reference. Check if we want to recurse. + let is_walk_needed = |mplace: &MPlaceTy<'tcx>| -> InterpResult<'tcx, bool> { + // ZSTs cannot contain pointers, we can avoid the interning walk. + if mplace.layout.is_zst() { + return Ok(false); + } + + // Now, check whether this allocation could contain references. + // + // Note, this check may sometimes not be cheap, so we only do it when the walk we'd like + // to avoid could be expensive: on the potentially larger types, arrays and slices, + // rather than on all aggregates unconditionally. + if matches!(mplace.layout.ty.kind(), ty::Array(..) | ty::Slice(..)) { + let Some((size, align)) = self.ecx.size_and_align_of_mplace(&mplace)? else { + // We do the walk if we can't determine the size of the mplace: we may be + // dealing with extern types here in the future. + return Ok(true); + }; + + // If there is no provenance in this allocation, it does not contain references + // that point to another allocation, and we can avoid the interning walk. + if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, align)? { + if !alloc.has_provenance() { + return Ok(false); + } + } else { + // We're encountering a ZST here, and can avoid the walk as well. + return Ok(false); + } + } + + // In the general case, we do the walk. + Ok(true) + }; + + // If this allocation contains no references to intern, we avoid the potentially costly + // walk. + // + // We can do this before the checks for interior mutability below, because only references + // are relevant in that situation, and we're checking if there are any here. + if !is_walk_needed(mplace)? { + return Ok(()); + } + + if let Some(def) = mplace.layout.ty.ty_adt_def() { + if def.is_unsafe_cell() { + // We are crossing over an `UnsafeCell`, we can mutate again. This means that + // References we encounter inside here are interned as pointing to mutable + // allocations. + // Remember the `old` value to handle nested `UnsafeCell`. + let old = std::mem::replace(&mut self.inside_unsafe_cell, true); + let walked = self.walk_value(mplace); + self.inside_unsafe_cell = old; + return walked; + } + } + self.walk_value(mplace) } } diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 04cae23f852a1..3f69716828053 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -226,8 +226,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } sym::discriminant_value => { let place = self.deref_operand(&args[0])?; - let discr_val = self.read_discriminant(&place.into())?.0; - self.write_scalar(discr_val, dest)?; + let variant = self.read_discriminant(&place.into())?; + let discr = self.discriminant_for_variant(place.layout, variant)?; + self.write_scalar(discr, dest)?; } sym::exact_div => { let l = self.read_immediate(&args[0])?; @@ -425,11 +426,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ); for i in 0..dest_len { - let place = self.mplace_index(&dest, i)?; + let place = self.project_index(&dest, i)?; let value = if i == index { elem.clone() } else { - self.mplace_index(&input, i)?.into() + self.project_index(&input, i)?.into() }; self.copy_op(&value, &place.into(), /*allow_transmute*/ false)?; } @@ -444,7 +445,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { input_len ); self.copy_op( - &self.mplace_index(&input, index)?.into(), + &self.project_index(&input, index)?.into(), dest, /*allow_transmute*/ false, )?; diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs index c4fe293bfaccc..44a12751743dc 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs @@ -101,11 +101,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let location = self.allocate(loc_layout, MemoryKind::CallerLocation).unwrap(); // Initialize fields. - self.write_immediate(file.to_ref(self), &self.mplace_field(&location, 0).unwrap().into()) + self.write_immediate(file.to_ref(self), &self.project_field(&location, 0).unwrap().into()) .expect("writing to memory we just allocated cannot fail"); - self.write_scalar(line, &self.mplace_field(&location, 1).unwrap().into()) + self.write_scalar(line, &self.project_field(&location, 1).unwrap().into()) .expect("writing to memory we just allocated cannot fail"); - self.write_scalar(col, &self.mplace_field(&location, 2).unwrap().into()) + self.write_scalar(col, &self.project_field(&location, 2).unwrap().into()) .expect("writing to memory we just allocated cannot fail"); location diff --git a/compiler/rustc_const_eval/src/interpret/mod.rs b/compiler/rustc_const_eval/src/interpret/mod.rs index f657f954f9c6f..7974920bc14da 100644 --- a/compiler/rustc_const_eval/src/interpret/mod.rs +++ b/compiler/rustc_const_eval/src/interpret/mod.rs @@ -26,9 +26,10 @@ pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackP pub use self::memory::{AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind}; pub use self::operand::{ImmTy, Immediate, OpTy, Operand}; pub use self::place::{MPlaceTy, MemPlace, MemPlaceMeta, Place, PlaceTy}; +pub use self::projection::Projectable; pub use self::terminator::FnArg; pub use self::validity::{CtfeValidationMode, RefTracking}; -pub use self::visitor::{MutValueVisitor, Value, ValueVisitor}; +pub use self::visitor::ValueVisitor; pub(crate) use self::intrinsics::eval_nullary_intrinsic; use eval_context::{from_known_layout, mir_assign_valid_types}; diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 47383c0eabc3b..be39e63ab4f5a 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -1,6 +1,8 @@ //! Functions concerning immediate values and operands, and reading from operands. //! All high-level functions to read from memory work on operands as sources. +use std::assert_matches::assert_matches; + use either::{Either, Left, Right}; use rustc_hir::def::Namespace; @@ -13,8 +15,8 @@ use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size}; use super::{ alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstValue, Frame, GlobalId, - InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, Place, PlaceTy, Pointer, - Provenance, Scalar, + InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, PlaceTy, Pointer, + Projectable, Provenance, Scalar, }; /// An `Immediate` represents a single immediate self-contained Rust value. @@ -199,6 +201,20 @@ impl<'tcx, Prov: Provenance> From> for OpTy<'tcx, Prov> { } } +impl<'tcx, Prov: Provenance> From<&'_ ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> { + #[inline(always)] + fn from(val: &ImmTy<'tcx, Prov>) -> Self { + OpTy { op: Operand::Immediate(val.imm), layout: val.layout, align: None } + } +} + +impl<'tcx, Prov: Provenance> From<&'_ mut ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> { + #[inline(always)] + fn from(val: &mut ImmTy<'tcx, Prov>) -> Self { + OpTy { op: Operand::Immediate(val.imm), layout: val.layout, align: None } + } +} + impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { #[inline] pub fn from_scalar(val: Scalar, layout: TyAndLayout<'tcx>) -> Self { @@ -240,43 +256,128 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { let int = self.to_scalar().assert_int(); ConstInt::new(int, self.layout.ty.is_signed(), self.layout.ty.is_ptr_sized_integral()) } + + /// Compute the "sub-immediate" that is located within the `base` at the given offset with the + /// given layout. + // Not called `offset` to avoid confusion with the trait method. + fn offset_(&self, offset: Size, layout: TyAndLayout<'tcx>, cx: &impl HasDataLayout) -> Self { + // This makes several assumptions about what layouts we will encounter; we match what + // codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`). + let inner_val: Immediate<_> = match (**self, self.layout.abi) { + // if the entire value is uninit, then so is the field (can happen in ConstProp) + (Immediate::Uninit, _) => Immediate::Uninit, + // the field contains no information, can be left uninit + _ if layout.is_zst() => Immediate::Uninit, + // some fieldless enum variants can have non-zero size but still `Aggregate` ABI... try + // to detect those here and also give them no data + _ if matches!(layout.abi, Abi::Aggregate { .. }) + && matches!(&layout.fields, abi::FieldsShape::Arbitrary { offsets, .. } if offsets.len() == 0) => + { + Immediate::Uninit + } + // the field covers the entire type + _ if layout.size == self.layout.size => { + assert_eq!(offset.bytes(), 0); + assert!( + match (self.layout.abi, layout.abi) { + (Abi::Scalar(..), Abi::Scalar(..)) => true, + (Abi::ScalarPair(..), Abi::ScalarPair(..)) => true, + _ => false, + }, + "cannot project into {} immediate with equally-sized field {}\nouter ABI: {:#?}\nfield ABI: {:#?}", + self.layout.ty, + layout.ty, + self.layout.abi, + layout.abi, + ); + **self + } + // extract fields from types with `ScalarPair` ABI + (Immediate::ScalarPair(a_val, b_val), Abi::ScalarPair(a, b)) => { + assert!(matches!(layout.abi, Abi::Scalar(..))); + Immediate::from(if offset.bytes() == 0 { + debug_assert_eq!(layout.size, a.size(cx)); + a_val + } else { + debug_assert_eq!(offset, a.size(cx).align_to(b.align(cx).abi)); + debug_assert_eq!(layout.size, b.size(cx)); + b_val + }) + } + // everything else is a bug + _ => bug!("invalid field access on immediate {}, layout {:#?}", self, self.layout), + }; + + ImmTy::from_immediate(inner_val, layout) + } +} + +impl<'mir, 'tcx: 'mir, Prov: Provenance> Projectable<'mir, 'tcx, Prov> for ImmTy<'tcx, Prov> { + #[inline(always)] + fn layout(&self) -> TyAndLayout<'tcx> { + self.layout + } + + fn meta>( + &self, + _ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, MemPlaceMeta> { + assert!(self.layout.is_sized()); // unsized ImmTy can only exist temporarily and should never reach this here + Ok(MemPlaceMeta::None) + } + + fn offset_with_meta( + &self, + offset: Size, + meta: MemPlaceMeta, + layout: TyAndLayout<'tcx>, + cx: &impl HasDataLayout, + ) -> InterpResult<'tcx, Self> { + assert_matches!(meta, MemPlaceMeta::None); // we can't store this anywhere anyway + Ok(self.offset_(offset, layout, cx)) + } + + fn to_op>( + &self, + _ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + Ok(self.into()) + } } impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> { - pub fn len(&self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> { - if self.layout.is_unsized() { - if matches!(self.op, Operand::Immediate(Immediate::Uninit)) { - // Uninit unsized places shouldn't occur. In the interpreter we have them - // temporarily for unsized arguments before their value is put in; in ConstProp they - // remain uninit and this code can actually be reached. - throw_inval!(UninitUnsizedLocal); + // Provided as inherent method since it doesn't need the `ecx` of `Projectable::meta`. + pub fn meta(&self) -> InterpResult<'tcx, MemPlaceMeta> { + Ok(if self.layout.is_unsized() { + if matches!(self.op, Operand::Immediate(_)) { + // Unsized immediate OpTy cannot occur. We create a MemPlace for all unsized locals during argument passing. + // However, ConstProp doesn't do that, so we can run into this nonsense situation. + throw_inval!(ConstPropNonsense); } // There are no unsized immediates. - self.assert_mem_place().len(cx) + self.assert_mem_place().meta } else { - match self.layout.fields { - abi::FieldsShape::Array { count, .. } => Ok(count), - _ => bug!("len not supported on sized type {:?}", self.layout.ty), - } - } + MemPlaceMeta::None + }) } +} - /// Replace the layout of this operand. There's basically no sanity check that this makes sense, - /// you better know what you are doing! If this is an immediate, applying the wrong layout can - /// not just lead to invalid data, it can actually *shift the data around* since the offsets of - /// a ScalarPair are entirely determined by the layout, not the data. - pub fn transmute(&self, layout: TyAndLayout<'tcx>) -> Self { - assert_eq!( - self.layout.size, layout.size, - "transmuting with a size change, that doesn't seem right" - ); - OpTy { layout, ..*self } +impl<'mir, 'tcx: 'mir, Prov: Provenance + 'static> Projectable<'mir, 'tcx, Prov> + for OpTy<'tcx, Prov> +{ + #[inline(always)] + fn layout(&self) -> TyAndLayout<'tcx> { + self.layout } - /// Offset the operand in memory (if possible) and change its metadata. - /// - /// This can go wrong very easily if you give the wrong layout for the new place! - pub(super) fn offset_with_meta( + fn meta>( + &self, + _ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, MemPlaceMeta> { + self.meta() + } + + fn offset_with_meta( &self, offset: Size, meta: MemPlaceMeta, @@ -286,28 +387,18 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> { match self.as_mplace_or_imm() { Left(mplace) => Ok(mplace.offset_with_meta(offset, meta, layout, cx)?.into()), Right(imm) => { - assert!( - matches!(*imm, Immediate::Uninit), - "Scalar/ScalarPair cannot be offset into" - ); assert!(!meta.has_meta()); // no place to store metadata here // Every part of an uninit is uninit. - Ok(ImmTy::uninit(layout).into()) + Ok(imm.offset(offset, layout, cx)?.into()) } } } - /// Offset the operand in memory (if possible). - /// - /// This can go wrong very easily if you give the wrong layout for the new place! - pub fn offset( + fn to_op>( &self, - offset: Size, - layout: TyAndLayout<'tcx>, - cx: &impl HasDataLayout, - ) -> InterpResult<'tcx, Self> { - assert!(layout.is_sized()); - self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx) + _ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + Ok(self.clone()) } } @@ -497,18 +588,28 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Every place can be read from, so we can turn them into an operand. /// This will definitely return `Indirect` if the place is a `Ptr`, i.e., this /// will never actually read from memory. - #[inline(always)] pub fn place_to_op( &self, place: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - let op = match **place { - Place::Ptr(mplace) => Operand::Indirect(mplace), - Place::Local { frame, local } => { - *self.local_to_op(&self.stack()[frame], local, None)? + match place.as_mplace_or_local() { + Left(mplace) => Ok(mplace.into()), + Right((frame, local, offset)) => { + let base = self.local_to_op(&self.stack()[frame], local, None)?; + let mut field = if let Some(offset) = offset { + // This got offset. We can be sure that the field is sized. + base.offset(offset, place.layout, self)? + } else { + assert_eq!(place.layout, base.layout); + // Unsized cases are possible here since an unsized local will be a + // `Place::Local` until the first projection calls `place_to_op` to extract the + // underlying mplace. + base + }; + field.align = Some(place.align); + Ok(field) } - }; - Ok(OpTy { op, layout: place.layout, align: Some(place.align) }) + } } /// Evaluate a place with the goal of reading from it. This lets us sometimes @@ -525,7 +626,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let mut op = self.local_to_op(self.frame(), mir_place.local, layout)?; // Using `try_fold` turned out to be bad for performance, hence the loop. for elem in mir_place.projection.iter() { - op = self.operand_projection(&op, elem)? + op = self.project(&op, elem)? } trace!("eval_place_to_op: got {:?}", *op); diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs index e04764636cc14..49c3b152e1d7f 100644 --- a/compiler/rustc_const_eval/src/interpret/operator.rs +++ b/compiler/rustc_const_eval/src/interpret/operator.rs @@ -38,9 +38,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // With randomized layout, `(int, bool)` might cease to be a `ScalarPair`, so we have to // do a component-wise write here. This code path is slower than the above because // `place_field` will have to `force_allocate` locals here. - let val_field = self.place_field(&dest, 0)?; + let val_field = self.project_field(dest, 0)?; self.write_scalar(val, &val_field)?; - let overflowed_field = self.place_field(&dest, 1)?; + let overflowed_field = self.project_field(dest, 1)?; self.write_scalar(Scalar::from_bool(overflowed), &overflowed_field)?; } Ok(()) diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index a9b2b43f1e6f5..db1239c7136aa 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -2,11 +2,14 @@ //! into a place. //! All high-level functions to write to memory work on places as destinations. +use std::assert_matches::assert_matches; + use either::{Either, Left, Right}; use rustc_ast::Mutability; use rustc_index::IndexSlice; use rustc_middle::mir; +use rustc_middle::mir::interpret::PointerArithmetic; use rustc_middle::ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::Ty; @@ -15,7 +18,7 @@ use rustc_target::abi::{self, Abi, Align, FieldIdx, HasDataLayout, Size, FIRST_V use super::{ alloc_range, mir_assign_valid_types, AllocId, AllocRef, AllocRefMut, CheckInAllocMsg, ConstAlloc, ImmTy, Immediate, InterpCx, InterpResult, Machine, MemoryKind, OpTy, Operand, - Pointer, Provenance, Scalar, + Pointer, Projectable, Provenance, Scalar, }; #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] @@ -44,6 +47,27 @@ impl MemPlaceMeta { Self::None => false, } } + + pub(crate) fn len<'tcx>( + &self, + layout: TyAndLayout<'tcx>, + cx: &impl HasDataLayout, + ) -> InterpResult<'tcx, u64> { + if layout.is_unsized() { + // We need to consult `meta` metadata + match layout.ty.kind() { + ty::Slice(..) | ty::Str => self.unwrap_meta().to_target_usize(cx), + _ => bug!("len not supported on unsized type {:?}", layout.ty), + } + } else { + // Go through the layout. There are lots of types that support a length, + // e.g., SIMD types. (But not all repr(simd) types even have FieldsShape::Array!) + match layout.fields { + abi::FieldsShape::Array { count, .. } => Ok(count), + _ => bug!("len not supported on sized type {:?}", layout.ty), + } + } + } } #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] @@ -73,9 +97,13 @@ pub enum Place { /// A place referring to a value allocated in the `Memory` system. Ptr(MemPlace), - /// To support alloc-free locals, we are able to write directly to a local. + /// To support alloc-free locals, we are able to write directly to a local. The offset indicates + /// where in the local this place is located; if it is `None`, no projection has been applied. + /// Such projections are meaningful even if the offset is 0, since they can change layouts. /// (Without that optimization, we'd just always be a `MemPlace`.) - Local { frame: usize, local: mir::Local }, + /// Note that this only stores the frame index, not the thread this frame belongs to -- that is + /// implicit. This means a `Place` must never be moved across interpreter thread boundaries! + Local { frame: usize, local: mir::Local, offset: Option }, } #[derive(Clone, Debug)] @@ -132,6 +160,11 @@ impl MemPlace { MemPlace { ptr, meta: MemPlaceMeta::None } } + #[inline(always)] + pub fn from_ptr_with_meta(ptr: Pointer>, meta: MemPlaceMeta) -> Self { + MemPlace { ptr, meta } + } + /// Adjust the provenance of the main pointer (metadata is unaffected). pub fn map_provenance(self, f: impl FnOnce(Option) -> Option) -> Self { MemPlace { ptr: self.ptr.map_provenance(f), ..self } @@ -150,7 +183,8 @@ impl MemPlace { } #[inline] - pub(super) fn offset_with_meta<'tcx>( + // Not called `offset_with_meta` to avoid confusion with the trait method. + fn offset_with_meta_<'tcx>( self, offset: Size, meta: MemPlaceMeta, @@ -164,19 +198,6 @@ impl MemPlace { } } -impl Place { - /// Asserts that this points to some local variable. - /// Returns the frame idx and the variable idx. - #[inline] - #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) - pub fn assert_local(&self) -> (usize, mir::Local) { - match self { - Place::Local { frame, local } => (*frame, *local), - _ => bug!("assert_local: expected Place::Local, got {:?}", self), - } - } -} - impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> { /// Produces a MemPlace that works for ZST but nothing else. /// Conceptually this is a new allocation, but it doesn't actually create an allocation so you @@ -189,37 +210,6 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> { MPlaceTy { mplace: MemPlace { ptr, meta: MemPlaceMeta::None }, layout, align } } - /// Offset the place in memory and change its metadata. - /// - /// This can go wrong very easily if you give the wrong layout for the new place! - #[inline] - pub(crate) fn offset_with_meta( - &self, - offset: Size, - meta: MemPlaceMeta, - layout: TyAndLayout<'tcx>, - cx: &impl HasDataLayout, - ) -> InterpResult<'tcx, Self> { - Ok(MPlaceTy { - mplace: self.mplace.offset_with_meta(offset, meta, cx)?, - align: self.align.restrict_for_offset(offset), - layout, - }) - } - - /// Offset the place in memory. - /// - /// This can go wrong very easily if you give the wrong layout for the new place! - pub fn offset( - &self, - offset: Size, - layout: TyAndLayout<'tcx>, - cx: &impl HasDataLayout, - ) -> InterpResult<'tcx, Self> { - assert!(layout.is_sized()); - self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx) - } - #[inline] pub fn from_aligned_ptr(ptr: Pointer>, layout: TyAndLayout<'tcx>) -> Self { MPlaceTy { mplace: MemPlace::from_ptr(ptr), layout, align: layout.align.abi } @@ -231,28 +221,48 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> { layout: TyAndLayout<'tcx>, meta: MemPlaceMeta, ) -> Self { - let mut mplace = MemPlace::from_ptr(ptr); - mplace.meta = meta; + MPlaceTy { + mplace: MemPlace::from_ptr_with_meta(ptr, meta), + layout, + align: layout.align.abi, + } + } +} - MPlaceTy { mplace, layout, align: layout.align.abi } +impl<'mir, 'tcx: 'mir, Prov: Provenance + 'static> Projectable<'mir, 'tcx, Prov> + for MPlaceTy<'tcx, Prov> +{ + #[inline(always)] + fn layout(&self) -> TyAndLayout<'tcx> { + self.layout } - #[inline] - pub(crate) fn len(&self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> { - if self.layout.is_unsized() { - // We need to consult `meta` metadata - match self.layout.ty.kind() { - ty::Slice(..) | ty::Str => self.mplace.meta.unwrap_meta().to_target_usize(cx), - _ => bug!("len not supported on unsized type {:?}", self.layout.ty), - } - } else { - // Go through the layout. There are lots of types that support a length, - // e.g., SIMD types. (But not all repr(simd) types even have FieldsShape::Array!) - match self.layout.fields { - abi::FieldsShape::Array { count, .. } => Ok(count), - _ => bug!("len not supported on sized type {:?}", self.layout.ty), - } - } + fn meta>( + &self, + _ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, MemPlaceMeta> { + Ok(self.meta) + } + + fn offset_with_meta( + &self, + offset: Size, + meta: MemPlaceMeta, + layout: TyAndLayout<'tcx>, + cx: &impl HasDataLayout, + ) -> InterpResult<'tcx, Self> { + Ok(MPlaceTy { + mplace: self.mplace.offset_with_meta_(offset, meta, cx)?, + align: self.align.restrict_for_offset(offset), + layout, + }) + } + + fn to_op>( + &self, + _ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + Ok(self.into()) } } @@ -280,13 +290,15 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> { } } -impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> { +impl<'tcx, Prov: Provenance + 'static> PlaceTy<'tcx, Prov> { /// A place is either an mplace or some local. #[inline] - pub fn as_mplace_or_local(&self) -> Either, (usize, mir::Local)> { + pub fn as_mplace_or_local( + &self, + ) -> Either, (usize, mir::Local, Option)> { match **self { Place::Ptr(mplace) => Left(MPlaceTy { mplace, layout: self.layout, align: self.align }), - Place::Local { frame, local } => Right((frame, local)), + Place::Local { frame, local, offset } => Right((frame, local, offset)), } } @@ -302,12 +314,76 @@ impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> { } } +impl<'mir, 'tcx: 'mir, Prov: Provenance + 'static> Projectable<'mir, 'tcx, Prov> + for PlaceTy<'tcx, Prov> +{ + #[inline(always)] + fn layout(&self) -> TyAndLayout<'tcx> { + self.layout + } + + fn meta>( + &self, + ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, MemPlaceMeta> { + ecx.place_meta(self) + } + + fn offset_with_meta( + &self, + offset: Size, + meta: MemPlaceMeta, + layout: TyAndLayout<'tcx>, + cx: &impl HasDataLayout, + ) -> InterpResult<'tcx, Self> { + Ok(match self.as_mplace_or_local() { + Left(mplace) => mplace.offset_with_meta(offset, meta, layout, cx)?.into(), + Right((frame, local, old_offset)) => { + assert_matches!(meta, MemPlaceMeta::None); // we couldn't store it anyway... + let new_offset = cx + .data_layout() + .offset(old_offset.unwrap_or(Size::ZERO).bytes(), offset.bytes())?; + PlaceTy { + place: Place::Local { + frame, + local, + offset: Some(Size::from_bytes(new_offset)), + }, + align: self.align.restrict_for_offset(offset), + layout, + } + } + }) + } + + fn to_op>( + &self, + ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + ecx.place_to_op(self) + } +} + // FIXME: Working around https://github.com/rust-lang/rust/issues/54385 impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M> where Prov: Provenance + 'static, M: Machine<'mir, 'tcx, Provenance = Prov>, { + /// Get the metadata of the given place. + pub(super) fn place_meta( + &self, + place: &PlaceTy<'tcx, M::Provenance>, + ) -> InterpResult<'tcx, MemPlaceMeta> { + if place.layout.is_unsized() { + // For `Place::Local`, the metadata is stored with the local, not the place. So we have + // to look that up first. + self.place_to_op(place)?.meta() + } else { + Ok(MemPlaceMeta::None) + } + } + /// Take a value, which represents a (thin or wide) reference, and make it a place. /// Alignment is just based on the type. This is the inverse of `MemPlace::to_ref()`. /// @@ -327,11 +403,9 @@ where Immediate::Uninit => throw_ub!(InvalidUninitBytes(None)), }; - let mplace = MemPlace { ptr: ptr.to_pointer(self)?, meta }; // `ref_to_mplace` is called on raw pointers even if they don't actually get dereferenced; // we hence can't call `size_and_align_of` since that asserts more validity than we want. - let align = layout.align.abi; - Ok(MPlaceTy { mplace, layout, align }) + Ok(MPlaceTy::from_aligned_ptr_with_meta(ptr.to_pointer(self)?, layout, meta)) } /// Take an operand, representing a pointer, and dereference it to a place. @@ -422,7 +496,7 @@ where local: mir::Local, ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { let layout = self.layout_of_local(&self.stack()[frame], local, None)?; - let place = Place::Local { frame, local }; + let place = Place::Local { frame, local, offset: None }; Ok(PlaceTy { place, layout, align: layout.align.abi }) } @@ -430,13 +504,13 @@ where /// place; for reading, a more efficient alternative is `eval_place_to_op`. #[instrument(skip(self), level = "debug")] pub fn eval_place( - &mut self, + &self, mir_place: mir::Place<'tcx>, ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { let mut place = self.local_to_place(self.frame_idx(), mir_place.local)?; // Using `try_fold` turned out to be bad for performance, hence the loop. for elem in mir_place.projection.iter() { - place = self.place_projection(&place, elem)? + place = self.project(&place, elem)? } trace!("{:?}", self.dump_place(place.place)); @@ -503,22 +577,54 @@ where src: Immediate, dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx> { - assert!(dest.layout.is_sized(), "Cannot write unsized data"); + assert!(dest.layout.is_sized(), "Cannot write unsized immediate data"); trace!("write_immediate: {:?} <- {:?}: {}", *dest, src, dest.layout.ty); // See if we can avoid an allocation. This is the counterpart to `read_immediate_raw`, // but not factored as a separate function. let mplace = match dest.place { - Place::Local { frame, local } => { - match M::access_local_mut(self, frame, local)? { - Operand::Immediate(local) => { - // Local can be updated in-place. - *local = src; - return Ok(()); - } - Operand::Indirect(mplace) => { - // The local is in memory, go on below. - *mplace + Place::Local { frame, local, offset } => { + if offset.is_some() { + // This has been projected to a part of this local. We could have complicated + // logic to still keep this local as an `Operand`... but it's much easier to + // just fall back to the indirect path. + *self.force_allocation(dest)? + } else { + match M::access_local_mut(self, frame, local)? { + Operand::Immediate(local_val) => { + // Local can be updated in-place. + *local_val = src; + // Double-check that the value we are storing and the local fit to each other. + // (*After* doing the update for borrow checker reasons.) + if cfg!(debug_assertions) { + let local_layout = + self.layout_of_local(&self.stack()[frame], local, None)?; + match (src, local_layout.abi) { + (Immediate::Scalar(scalar), Abi::Scalar(s)) => { + assert_eq!(scalar.size(), s.size(self)) + } + ( + Immediate::ScalarPair(a_val, b_val), + Abi::ScalarPair(a, b), + ) => { + assert_eq!(a_val.size(), a.size(self)); + assert_eq!(b_val.size(), b.size(self)); + } + (Immediate::Uninit, _) => {} + (src, abi) => { + bug!( + "value {src:?} cannot be written into local with type {} (ABI {abi:?})", + local_layout.ty + ) + } + }; + } + return Ok(()); + } + Operand::Indirect(mplace) => { + // The local is in memory, go on below. + *mplace + } } } } @@ -593,15 +699,23 @@ where pub fn write_uninit(&mut self, dest: &PlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx> { let mplace = match dest.as_mplace_or_local() { Left(mplace) => mplace, - Right((frame, local)) => { - match M::access_local_mut(self, frame, local)? { - Operand::Immediate(local) => { - *local = Immediate::Uninit; - return Ok(()); - } - Operand::Indirect(mplace) => { - // The local is in memory, go on below. - MPlaceTy { mplace: *mplace, layout: dest.layout, align: dest.align } + Right((frame, local, offset)) => { + if offset.is_some() { + // This has been projected to a part of this local. We could have complicated + // logic to still keep this local as an `Operand`... but it's much easier to + // just fall back to the indirect path. + // FIXME: share the logic with `write_immediate_no_validate`. + self.force_allocation(dest)? + } else { + match M::access_local_mut(self, frame, local)? { + Operand::Immediate(local) => { + *local = Immediate::Uninit; + return Ok(()); + } + Operand::Indirect(mplace) => { + // The local is in memory, go on below. + MPlaceTy { mplace: *mplace, layout: dest.layout, align: dest.align } + } } } } @@ -728,8 +842,8 @@ where place: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> { let mplace = match place.place { - Place::Local { frame, local } => { - match M::access_local_mut(self, frame, local)? { + Place::Local { frame, local, offset } => { + let whole_local = match M::access_local_mut(self, frame, local)? { &mut Operand::Immediate(local_val) => { // We need to make an allocation. @@ -742,10 +856,11 @@ where throw_unsup_format!("unsized locals are not supported"); } let mplace = *self.allocate(local_layout, MemoryKind::Stack)?; + // Preserve old value. (As an optimization, we can skip this if it was uninit.) if !matches!(local_val, Immediate::Uninit) { - // Preserve old value. (As an optimization, we can skip this if it was uninit.) - // We don't have to validate as we can assume the local - // was already valid for its type. + // We don't have to validate as we can assume the local was already + // valid for its type. We must not use any part of `place` here, that + // could be a projection to a part of the local! self.write_immediate_to_mplace_no_validate( local_val, local_layout, @@ -753,18 +868,25 @@ where mplace, )?; } - // Now we can call `access_mut` again, asserting it goes well, - // and actually overwrite things. + // Now we can call `access_mut` again, asserting it goes well, and actually + // overwrite things. This points to the entire allocation, not just the part + // the place refers to, i.e. we do this before we apply `offset`. *M::access_local_mut(self, frame, local).unwrap() = Operand::Indirect(mplace); mplace } &mut Operand::Indirect(mplace) => mplace, // this already was an indirect local + }; + if let Some(offset) = offset { + whole_local.offset_with_meta_(offset, MemPlaceMeta::None, self)? + } else { + // Preserve wide place metadata, do not call `offset`. + whole_local } } Place::Ptr(mplace) => mplace, }; - // Return with the original layout, so that the caller can go on + // Return with the original layout and align, so that the caller can go on Ok(MPlaceTy { mplace, layout: place.layout, align: place.align }) } @@ -809,7 +931,7 @@ where self.write_uninit(&dest)?; let (variant_index, variant_dest, active_field_index) = match *kind { mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => { - let variant_dest = self.place_downcast(&dest, variant_index)?; + let variant_dest = self.project_downcast(dest, variant_index)?; (variant_index, variant_dest, active_field_index) } _ => (FIRST_VARIANT, dest.clone(), None), @@ -819,7 +941,7 @@ where } for (field_index, operand) in operands.iter_enumerated() { let field_index = active_field_index.unwrap_or(field_index); - let field_dest = self.place_field(&variant_dest, field_index.as_usize())?; + let field_dest = self.project_field(&variant_dest, field_index.as_usize())?; let op = self.eval_operand(operand, Some(field_dest.layout))?; self.copy_op(&op, &field_dest, /*allow_transmute*/ false)?; } @@ -859,22 +981,24 @@ where Ok((mplace, vtable)) } - /// Turn an operand with a `dyn* Trait` type into an operand with the actual dynamic type. - /// Aso returns the vtable. - pub(super) fn unpack_dyn_star( + /// Turn a `dyn* Trait` type into an value with the actual dynamic type. + /// Also returns the vtable. + pub(super) fn unpack_dyn_star>( &self, - op: &OpTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, (OpTy<'tcx, M::Provenance>, Pointer>)> { + val: &P, + ) -> InterpResult<'tcx, (P, Pointer>)> { assert!( - matches!(op.layout.ty.kind(), ty::Dynamic(_, _, ty::DynStar)), + matches!(val.layout().ty.kind(), ty::Dynamic(_, _, ty::DynStar)), "`unpack_dyn_star` only makes sense on `dyn*` types" ); - let data = self.operand_field(&op, 0)?; - let vtable = self.operand_field(&op, 1)?; - let vtable = self.read_pointer(&vtable)?; + let data = self.project_field(val, 0)?; + let vtable = self.project_field(val, 1)?; + let vtable = self.read_pointer(&vtable.to_op(self)?)?; let (ty, _) = self.get_ptr_vtable(vtable)?; let layout = self.layout_of(ty)?; - let data = data.transmute(layout); + // `data` is already the right thing but has the wrong type. So we transmute it, by + // projecting with offset 0. + let data = data.transmute(layout, self)?; Ok((data, vtable)) } } diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs index d7d31fe188757..ddcbc8350aaee 100644 --- a/compiler/rustc_const_eval/src/interpret/projection.rs +++ b/compiler/rustc_const_eval/src/interpret/projection.rs @@ -7,18 +7,71 @@ //! but we still need to do bounds checking and adjust the layout. To not duplicate that with MPlaceTy, we actually //! implement the logic on OpTy, and MPlaceTy calls that. -use either::{Left, Right}; - use rustc_middle::mir; use rustc_middle::ty; -use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::Ty; -use rustc_target::abi::{self, Abi, VariantIdx}; +use rustc_middle::ty::TyCtxt; +use rustc_target::abi::HasDataLayout; +use rustc_target::abi::Size; +use rustc_target::abi::{self, VariantIdx}; + +use super::MPlaceTy; +use super::{InterpCx, InterpResult, Machine, MemPlaceMeta, OpTy, Provenance, Scalar}; -use super::{ - ImmTy, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, PlaceTy, - Provenance, Scalar, -}; +/// A thing that we can project into, and that has a layout. +pub trait Projectable<'mir, 'tcx: 'mir, Prov: Provenance>: Sized { + /// Get the layout. + fn layout(&self) -> TyAndLayout<'tcx>; + + /// Get the metadata of a wide value. + fn meta>( + &self, + ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, MemPlaceMeta>; + + fn len>( + &self, + ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, u64> { + self.meta(ecx)?.len(self.layout(), ecx) + } + + /// Offset the value by the given amount, replacing the layout and metadata. + fn offset_with_meta( + &self, + offset: Size, + meta: MemPlaceMeta, + layout: TyAndLayout<'tcx>, + cx: &impl HasDataLayout, + ) -> InterpResult<'tcx, Self>; + + fn offset( + &self, + offset: Size, + layout: TyAndLayout<'tcx>, + cx: &impl HasDataLayout, + ) -> InterpResult<'tcx, Self> { + assert!(layout.is_sized()); + self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx) + } + + fn transmute( + &self, + layout: TyAndLayout<'tcx>, + cx: &impl HasDataLayout, + ) -> InterpResult<'tcx, Self> { + assert_eq!(self.layout().size, layout.size); + self.offset_with_meta(Size::ZERO, MemPlaceMeta::None, layout, cx) + } + + /// Convert this to an `OpTy`. This might be an irreversible transformation, but is useful for + /// reading from this thing. + fn to_op>( + &self, + ecx: &InterpCx<'mir, 'tcx, M>, + ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>>; +} // FIXME: Working around https://github.com/rust-lang/rust/issues/54385 impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M> @@ -26,167 +79,83 @@ where Prov: Provenance + 'static, M: Machine<'mir, 'tcx, Provenance = Prov>, { - //# Field access - /// Offset a pointer to project to a field of a struct/union. Unlike `place_field`, this is /// always possible without allocating, so it can take `&self`. Also return the field's layout. - /// This supports both struct and array fields. + /// This supports both struct and array fields, but not slices! /// /// This also works for arrays, but then the `usize` index type is restricting. /// For indexing into arrays, use `mplace_index`. - pub fn mplace_field( + pub fn project_field>( &self, - base: &MPlaceTy<'tcx, M::Provenance>, + base: &P, field: usize, - ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> { - let offset = base.layout.fields.offset(field); - let field_layout = base.layout.field(self, field); + ) -> InterpResult<'tcx, P> { + // Slices nominally have length 0, so they will panic somewhere in `fields.offset`. + debug_assert!( + !matches!(base.layout().ty.kind(), ty::Slice(..)), + "`field` projection called on a slice -- call `index` projection instead" + ); + let offset = base.layout().fields.offset(field); + let field_layout = base.layout().field(self, field); // Offset may need adjustment for unsized fields. let (meta, offset) = if field_layout.is_unsized() { + if base.layout().is_sized() { + // An unsized field of a sized type? Sure... + // But const-prop actually feeds us such nonsense MIR! + throw_inval!(ConstPropNonsense); + } + let base_meta = base.meta(self)?; // Re-use parent metadata to determine dynamic field layout. // With custom DSTS, this *will* execute user-defined code, but the same // happens at run-time so that's okay. - match self.size_and_align_of(&base.meta, &field_layout)? { - Some((_, align)) => (base.meta, offset.align_to(align)), + match self.size_and_align_of(&base_meta, &field_layout)? { + Some((_, align)) => (base_meta, offset.align_to(align)), None => { // For unsized types with an extern type tail we perform no adjustments. // NOTE: keep this in sync with `PlaceRef::project_field` in the codegen backend. - assert!(matches!(base.meta, MemPlaceMeta::None)); - (base.meta, offset) + assert!(matches!(base_meta, MemPlaceMeta::None)); + (base_meta, offset) } } } else { - // base.meta could be present; we might be accessing a sized field of an unsized + // base_meta could be present; we might be accessing a sized field of an unsized // struct. (MemPlaceMeta::None, offset) }; - // We do not look at `base.layout.align` nor `field_layout.align`, unlike - // codegen -- mostly to see if we can get away with that base.offset_with_meta(offset, meta, field_layout, self) } - /// Gets the place of a field inside the place, and also the field's type. - /// Just a convenience function, but used quite a bit. - /// This is the only projection that might have a side-effect: We cannot project - /// into the field of a local `ScalarPair`, we have to first allocate it. - pub fn place_field( - &mut self, - base: &PlaceTy<'tcx, M::Provenance>, - field: usize, - ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { - // FIXME: We could try to be smarter and avoid allocation for fields that span the - // entire place. - let base = self.force_allocation(base)?; - Ok(self.mplace_field(&base, field)?.into()) - } - - pub fn operand_field( + /// Downcasting to an enum variant. + pub fn project_downcast>( &self, - base: &OpTy<'tcx, M::Provenance>, - field: usize, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - let base = match base.as_mplace_or_imm() { - Left(ref mplace) => { - // We can reuse the mplace field computation logic for indirect operands. - let field = self.mplace_field(mplace, field)?; - return Ok(field.into()); - } - Right(value) => value, - }; - - let field_layout = base.layout.field(self, field); - let offset = base.layout.fields.offset(field); - // This makes several assumptions about what layouts we will encounter; we match what - // codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`). - let field_val: Immediate<_> = match (*base, base.layout.abi) { - // if the entire value is uninit, then so is the field (can happen in ConstProp) - (Immediate::Uninit, _) => Immediate::Uninit, - // the field contains no information, can be left uninit - _ if field_layout.is_zst() => Immediate::Uninit, - // the field covers the entire type - _ if field_layout.size == base.layout.size => { - assert!(match (base.layout.abi, field_layout.abi) { - (Abi::Scalar(..), Abi::Scalar(..)) => true, - (Abi::ScalarPair(..), Abi::ScalarPair(..)) => true, - _ => false, - }); - assert!(offset.bytes() == 0); - *base - } - // extract fields from types with `ScalarPair` ABI - (Immediate::ScalarPair(a_val, b_val), Abi::ScalarPair(a, b)) => { - assert!(matches!(field_layout.abi, Abi::Scalar(..))); - Immediate::from(if offset.bytes() == 0 { - debug_assert_eq!(field_layout.size, a.size(self)); - a_val - } else { - debug_assert_eq!(offset, a.size(self).align_to(b.align(self).abi)); - debug_assert_eq!(field_layout.size, b.size(self)); - b_val - }) - } - // everything else is a bug - _ => span_bug!( - self.cur_span(), - "invalid field access on immediate {}, layout {:#?}", - base, - base.layout - ), - }; - - Ok(ImmTy::from_immediate(field_val, field_layout).into()) - } - - //# Downcasting - - pub fn mplace_downcast( - &self, - base: &MPlaceTy<'tcx, M::Provenance>, + base: &P, variant: VariantIdx, - ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> { + ) -> InterpResult<'tcx, P> { + assert!(!base.meta(self)?.has_meta()); // Downcasts only change the layout. // (In particular, no check about whether this is even the active variant -- that's by design, // see https://github.com/rust-lang/rust/issues/93688#issuecomment-1032929496.) - assert!(!base.meta.has_meta()); - let mut base = *base; - base.layout = base.layout.for_variant(self, variant); - Ok(base) - } - - pub fn place_downcast( - &self, - base: &PlaceTy<'tcx, M::Provenance>, - variant: VariantIdx, - ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { - // Downcast just changes the layout - let mut base = base.clone(); - base.layout = base.layout.for_variant(self, variant); - Ok(base) - } - - pub fn operand_downcast( - &self, - base: &OpTy<'tcx, M::Provenance>, - variant: VariantIdx, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - // Downcast just changes the layout - let mut base = base.clone(); - base.layout = base.layout.for_variant(self, variant); - Ok(base) + // So we just "offset" by 0. + let layout = base.layout().for_variant(self, variant); + if layout.abi.is_uninhabited() { + // `read_discriminant` should have excluded uninhabited variants... but ConstProp calls + // us on dead code. + throw_inval!(ConstPropNonsense) + } + // This cannot be `transmute` as variants *can* have a smaller size than the entire enum. + base.offset(Size::ZERO, layout, self) } - //# Slice indexing - - #[inline(always)] - pub fn operand_index( + /// Compute the offset and field layout for accessing the given index. + pub fn project_index>( &self, - base: &OpTy<'tcx, M::Provenance>, + base: &P, index: u64, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + ) -> InterpResult<'tcx, P> { // Not using the layout method because we want to compute on u64 - match base.layout.fields { + let (offset, field_layout) = match base.layout().fields { abi::FieldsShape::Array { stride, count: _ } => { // `count` is nonsense for slices, use the dynamic length instead. let len = base.len(self)?; @@ -196,63 +165,26 @@ where } let offset = stride * index; // `Size` multiplication // All fields have the same layout. - let field_layout = base.layout.field(self, 0); - base.offset(offset, field_layout, self) + let field_layout = base.layout().field(self, 0); + (offset, field_layout) } _ => span_bug!( self.cur_span(), "`mplace_index` called on non-array type {:?}", - base.layout.ty + base.layout().ty ), - } - } - - /// Iterates over all fields of an array. Much more efficient than doing the - /// same by repeatedly calling `operand_index`. - pub fn operand_array_fields<'a>( - &self, - base: &'a OpTy<'tcx, Prov>, - ) -> InterpResult<'tcx, impl Iterator>> + 'a> { - let len = base.len(self)?; // also asserts that we have a type where this makes sense - let abi::FieldsShape::Array { stride, .. } = base.layout.fields else { - span_bug!(self.cur_span(), "operand_array_fields: expected an array layout"); }; - let field_layout = base.layout.field(self, 0); - let dl = &self.tcx.data_layout; - // `Size` multiplication - Ok((0..len).map(move |i| base.offset(stride * i, field_layout, dl))) - } - - /// Index into an array. - pub fn mplace_index( - &self, - base: &MPlaceTy<'tcx, M::Provenance>, - index: u64, - ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> { - Ok(self.operand_index(&base.into(), index)?.assert_mem_place()) - } - pub fn place_index( - &mut self, - base: &PlaceTy<'tcx, M::Provenance>, - index: u64, - ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { - // There's not a lot we can do here, since we cannot have a place to a part of a local. If - // we are accessing the only element of a 1-element array, it's still the entire local... - // that doesn't seem worth it. - let base = self.force_allocation(base)?; - Ok(self.mplace_index(&base, index)?.into()) + base.offset(offset, field_layout, self) } - //# ConstantIndex support - - fn operand_constant_index( + fn project_constant_index>( &self, - base: &OpTy<'tcx, M::Provenance>, + base: &P, offset: u64, min_length: u64, from_end: bool, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + ) -> InterpResult<'tcx, P> { let n = base.len(self)?; if n < min_length { // This can only be reached in ConstProp and non-rustc-MIR. @@ -267,32 +199,38 @@ where offset }; - self.operand_index(base, index) + self.project_index(base, index) } - fn place_constant_index( - &mut self, - base: &PlaceTy<'tcx, M::Provenance>, - offset: u64, - min_length: u64, - from_end: bool, - ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { - let base = self.force_allocation(base)?; - Ok(self - .operand_constant_index(&base.into(), offset, min_length, from_end)? - .assert_mem_place() - .into()) + /// Iterates over all fields of an array. Much more efficient than doing the + /// same by repeatedly calling `operand_index`. + pub fn project_array_fields<'a, P: Projectable<'mir, 'tcx, M::Provenance>>( + &self, + base: &'a P, + ) -> InterpResult<'tcx, impl Iterator> + 'a> + where + 'tcx: 'a, + { + let abi::FieldsShape::Array { stride, .. } = base.layout().fields else { + span_bug!(self.cur_span(), "operand_array_fields: expected an array layout"); + }; + let len = base.len(self)?; + let field_layout = base.layout().field(self, 0); + let tcx: TyCtxt<'tcx> = *self.tcx; + // `Size` multiplication + Ok((0..len).map(move |i| { + base.offset_with_meta(stride * i, MemPlaceMeta::None, field_layout, &tcx) + })) } - //# Subslicing - - fn operand_subslice( + /// Subslicing + fn project_subslice>( &self, - base: &OpTy<'tcx, M::Provenance>, + base: &P, from: u64, to: u64, from_end: bool, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + ) -> InterpResult<'tcx, P> { let len = base.len(self)?; // also asserts that we have a type where this makes sense let actual_to = if from_end { if from.checked_add(to).map_or(true, |to| to > len) { @@ -306,16 +244,20 @@ where // Not using layout method because that works with usize, and does not work with slices // (that have count 0 in their layout). - let from_offset = match base.layout.fields { + let from_offset = match base.layout().fields { abi::FieldsShape::Array { stride, .. } => stride * from, // `Size` multiplication is checked _ => { - span_bug!(self.cur_span(), "unexpected layout of index access: {:#?}", base.layout) + span_bug!( + self.cur_span(), + "unexpected layout of index access: {:#?}", + base.layout() + ) } }; // Compute meta and new layout let inner_len = actual_to.checked_sub(from).unwrap(); - let (meta, ty) = match base.layout.ty.kind() { + let (meta, ty) = match base.layout().ty.kind() { // It is not nice to match on the type, but that seems to be the only way to // implement this. ty::Array(inner, _) => { @@ -323,85 +265,45 @@ where } ty::Slice(..) => { let len = Scalar::from_target_usize(inner_len, self); - (MemPlaceMeta::Meta(len), base.layout.ty) + (MemPlaceMeta::Meta(len), base.layout().ty) } _ => { - span_bug!(self.cur_span(), "cannot subslice non-array type: `{:?}`", base.layout.ty) + span_bug!( + self.cur_span(), + "cannot subslice non-array type: `{:?}`", + base.layout().ty + ) } }; let layout = self.layout_of(ty)?; - base.offset_with_meta(from_offset, meta, layout, self) - } - - pub fn place_subslice( - &mut self, - base: &PlaceTy<'tcx, M::Provenance>, - from: u64, - to: u64, - from_end: bool, - ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { - let base = self.force_allocation(base)?; - Ok(self.operand_subslice(&base.into(), from, to, from_end)?.assert_mem_place().into()) - } - - //# Applying a general projection - /// Projects into a place. - #[instrument(skip(self), level = "trace")] - pub fn place_projection( - &mut self, - base: &PlaceTy<'tcx, M::Provenance>, - proj_elem: mir::PlaceElem<'tcx>, - ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { - use rustc_middle::mir::ProjectionElem::*; - Ok(match proj_elem { - OpaqueCast(ty) => { - let mut place = base.clone(); - place.layout = self.layout_of(ty)?; - place - } - Field(field, _) => self.place_field(base, field.index())?, - Downcast(_, variant) => self.place_downcast(base, variant)?, - Deref => self.deref_operand(&self.place_to_op(base)?)?.into(), - Index(local) => { - let layout = self.layout_of(self.tcx.types.usize)?; - let n = self.local_to_op(self.frame(), local, Some(layout))?; - let n = self.read_target_usize(&n)?; - self.place_index(base, n)? - } - ConstantIndex { offset, min_length, from_end } => { - self.place_constant_index(base, offset, min_length, from_end)? - } - Subslice { from, to, from_end } => self.place_subslice(base, from, to, from_end)?, - }) + base.offset_with_meta(from_offset, meta, layout, self) } + /// Applying a general projection #[instrument(skip(self), level = "trace")] - pub fn operand_projection( - &self, - base: &OpTy<'tcx, M::Provenance>, - proj_elem: mir::PlaceElem<'tcx>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + pub fn project

(&self, base: &P, proj_elem: mir::PlaceElem<'tcx>) -> InterpResult<'tcx, P> + where + P: Projectable<'mir, 'tcx, M::Provenance> + + From> + + std::fmt::Debug, + { use rustc_middle::mir::ProjectionElem::*; Ok(match proj_elem { - OpaqueCast(ty) => { - let mut op = base.clone(); - op.layout = self.layout_of(ty)?; - op - } - Field(field, _) => self.operand_field(base, field.index())?, - Downcast(_, variant) => self.operand_downcast(base, variant)?, - Deref => self.deref_operand(base)?.into(), + OpaqueCast(ty) => base.transmute(self.layout_of(ty)?, self)?, + Field(field, _) => self.project_field(base, field.index())?, + Downcast(_, variant) => self.project_downcast(base, variant)?, + Deref => self.deref_operand(&base.to_op(self)?)?.into(), Index(local) => { let layout = self.layout_of(self.tcx.types.usize)?; let n = self.local_to_op(self.frame(), local, Some(layout))?; let n = self.read_target_usize(&n)?; - self.operand_index(base, n)? + self.project_index(base, n)? } ConstantIndex { offset, min_length, from_end } => { - self.operand_constant_index(base, offset, min_length, from_end)? + self.project_constant_index(base, offset, min_length, from_end)? } - Subslice { from, to, from_end } => self.operand_subslice(base, from, to, from_end)?, + Subslice { from, to, from_end } => self.project_subslice(base, from, to, from_end)?, }) } } diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 619da8abb7d22..9182d23128fe2 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -8,7 +8,7 @@ use rustc_middle::mir; use rustc_middle::mir::interpret::{InterpResult, Scalar}; use rustc_middle::ty::layout::LayoutOf; -use super::{ImmTy, InterpCx, Machine}; +use super::{ImmTy, InterpCx, Machine, Projectable}; use crate::util; impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { @@ -197,7 +197,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.get_place_alloc_mut(&dest)?; } else { // Write the src to the first element. - let first = self.mplace_field(&dest, 0)?; + let first = self.project_index(&dest, 0)?; self.copy_op(&src, &first.into(), /*allow_transmute*/ false)?; // This is performance-sensitive code for big static/const arrays! So we @@ -302,8 +302,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Discriminant(place) => { let op = self.eval_place_to_op(place, None)?; - let discr_val = self.read_discriminant(&op)?.0; - self.write_scalar(discr_val, &dest)?; + let variant = self.read_discriminant(&op)?; + let discr = self.discriminant_for_variant(op.layout, variant)?; + self.write_scalar(discr, &dest)?; } } diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index 7964c6be008cb..f934cca2517b9 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -60,13 +60,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } pub fn fn_arg_field( - &mut self, + &self, arg: &FnArg<'tcx, M::Provenance>, field: usize, ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> { Ok(match arg { - FnArg::Copy(op) => FnArg::Copy(self.operand_field(op, field)?), - FnArg::InPlace(place) => FnArg::InPlace(self.place_field(place, field)?), + FnArg::Copy(op) => FnArg::Copy(self.project_field(op, field)?), + FnArg::InPlace(place) => FnArg::InPlace(self.project_field(place, field)?), }) } @@ -239,7 +239,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Evaluate the arguments of a function call pub(super) fn eval_fn_call_arguments( - &mut self, + &self, ops: &[mir::Operand<'tcx>], ) -> InterpResult<'tcx, Vec>> { ops.iter() @@ -382,12 +382,16 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // This all has to be in memory, there are no immediate unsized values. let src = caller_arg_copy.assert_mem_place(); // The destination cannot be one of these "spread args". - let (dest_frame, dest_local) = callee_arg.assert_local(); + let (dest_frame, dest_local, dest_offset) = callee_arg + .as_mplace_or_local() + .right() + .expect("callee fn arguments must be locals"); // We are just initializing things, so there can't be anything here yet. assert!(matches!( *self.local_to_op(&self.stack()[dest_frame], dest_local, None)?, Operand::Immediate(Immediate::Uninit) )); + assert_eq!(dest_offset, None); // Allocate enough memory to hold `src`. let Some((size, align)) = self.size_and_align_of_mplace(&src)? else { span_bug!( @@ -595,7 +599,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { if Some(local) == body.spread_arg { // Must be a tuple for i in 0..dest.layout.fields.count() { - let dest = self.place_field(&dest, i)?; + let dest = self.project_field(&dest, i)?; let callee_abi = callee_args_abis.next().unwrap(); self.pass_argument(&mut caller_args, callee_abi, &dest)?; } @@ -677,7 +681,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Not there yet, search for the only non-ZST field. let mut non_zst_field = None; for i in 0..receiver.layout.fields.count() { - let field = self.operand_field(&receiver, i)?; + let field = self.project_field(&receiver, i)?; let zst = field.layout.is_zst() && field.layout.align.abi.bytes() == 1; if !zst { @@ -703,12 +707,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let (vptr, dyn_ty, adjusted_receiver) = if let ty::Dynamic(data, _, ty::DynStar) = receiver_place.layout.ty.kind() { - let (recv, vptr) = self.unpack_dyn_star(&receiver_place.into())?; + let (recv, vptr) = self.unpack_dyn_star(&receiver_place)?; let (dyn_ty, dyn_trait) = self.get_ptr_vtable(vptr)?; if dyn_trait != data.principal() { throw_ub_custom!(fluent::const_eval_dyn_star_call_vtable_mismatch); } - let recv = recv.assert_mem_place(); // we passed an MPlaceTy to `unpack_dyn_star` so we definitely still have one (vptr, dyn_ty, recv.ptr) } else { @@ -836,7 +839,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } ty::Dynamic(_, _, ty::DynStar) => { // Dropping a `dyn*`. Need to find actual drop fn. - self.unpack_dyn_star(&place.into())?.0.assert_mem_place() + self.unpack_dyn_star(&place)?.0 } _ => { debug_assert_eq!( diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index 21c655988a0e1..a82c98e720578 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -29,7 +29,7 @@ use std::hash::Hash; use super::UndefinedBehaviorInfo::*; use super::{ AllocId, CheckInAllocMsg, GlobalAlloc, ImmTy, Immediate, InterpCx, InterpResult, MPlaceTy, - Machine, MemPlaceMeta, OpTy, Pointer, Scalar, ValueVisitor, + Machine, MemPlaceMeta, OpTy, Pointer, Projectable, Scalar, ValueVisitor, }; macro_rules! throw_validation_failure { @@ -462,6 +462,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' /// Check if this is a value of primitive type, and if yes check the validity of the value /// at that type. Return `true` if the type is indeed primitive. + /// + /// Note that not all of these have `FieldsShape::Primitive`, e.g. wide references. fn try_visit_primitive( &mut self, value: &OpTy<'tcx, M::Provenance>, @@ -660,10 +662,9 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> InvalidTag(val) => InvalidEnumTag { value: format!("{val:x}"), }, - + UninhabitedEnumVariantRead(_) => UninhabitedEnumTag, InvalidUninitBytes(None) => UninitEnumTag, - ) - .1) + )) }) } @@ -733,60 +734,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> } } - // Recursively walk the value at its type. - self.walk_value(op)?; - - // *After* all of this, check the ABI. We need to check the ABI to handle - // types like `NonNull` where the `Scalar` info is more restrictive than what - // the fields say (`rustc_layout_scalar_valid_range_start`). - // But in most cases, this will just propagate what the fields say, - // and then we want the error to point at the field -- so, first recurse, - // then check ABI. - // - // FIXME: We could avoid some redundant checks here. For newtypes wrapping - // scalars, we do the same check on every "level" (e.g., first we check - // MyNewtype and then the scalar in there). - match op.layout.abi { - Abi::Uninhabited => { - let ty = op.layout.ty; - throw_validation_failure!(self.path, UninhabitedVal { ty }); - } - Abi::Scalar(scalar_layout) => { - if !scalar_layout.is_uninit_valid() { - // There is something to check here. - let scalar = self.read_scalar(op, ExpectedKind::InitScalar)?; - self.visit_scalar(scalar, scalar_layout)?; - } - } - Abi::ScalarPair(a_layout, b_layout) => { - // We can only proceed if *both* scalars need to be initialized. - // FIXME: find a way to also check ScalarPair when one side can be uninit but - // the other must be init. - if !a_layout.is_uninit_valid() && !b_layout.is_uninit_valid() { - let (a, b) = - self.read_immediate(op, ExpectedKind::InitScalar)?.to_scalar_pair(); - self.visit_scalar(a, a_layout)?; - self.visit_scalar(b, b_layout)?; - } - } - Abi::Vector { .. } => { - // No checks here, we assume layout computation gets this right. - // (This is harder to check since Miri does not represent these as `Immediate`. We - // also cannot use field projections since this might be a newtype around a vector.) - } - Abi::Aggregate { .. } => { - // Nothing to do. - } - } - - Ok(()) - } - - fn visit_aggregate( - &mut self, - op: &OpTy<'tcx, M::Provenance>, - fields: impl Iterator>, - ) -> InterpResult<'tcx> { + // Recursively walk the value at its type. Apply optimizations for some large types. match op.layout.ty.kind() { ty::Str => { let mplace = op.assert_mem_place(); // strings are unsized and hence never immediate @@ -874,12 +822,58 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> // ZST type, so either validation fails for all elements or none. ty::Array(tys, ..) | ty::Slice(tys) if self.ecx.layout_of(*tys)?.is_zst() => { // Validate just the first element (if any). - self.walk_aggregate(op, fields.take(1))? + if op.len(self.ecx)? > 0 { + self.visit_field(op, 0, &self.ecx.project_index(op, 0)?)?; + } } _ => { - self.walk_aggregate(op, fields)? // default handler + self.walk_value(op)?; // default handler } } + + // *After* all of this, check the ABI. We need to check the ABI to handle + // types like `NonNull` where the `Scalar` info is more restrictive than what + // the fields say (`rustc_layout_scalar_valid_range_start`). + // But in most cases, this will just propagate what the fields say, + // and then we want the error to point at the field -- so, first recurse, + // then check ABI. + // + // FIXME: We could avoid some redundant checks here. For newtypes wrapping + // scalars, we do the same check on every "level" (e.g., first we check + // MyNewtype and then the scalar in there). + match op.layout.abi { + Abi::Uninhabited => { + let ty = op.layout.ty; + throw_validation_failure!(self.path, UninhabitedVal { ty }); + } + Abi::Scalar(scalar_layout) => { + if !scalar_layout.is_uninit_valid() { + // There is something to check here. + let scalar = self.read_scalar(op, ExpectedKind::InitScalar)?; + self.visit_scalar(scalar, scalar_layout)?; + } + } + Abi::ScalarPair(a_layout, b_layout) => { + // We can only proceed if *both* scalars need to be initialized. + // FIXME: find a way to also check ScalarPair when one side can be uninit but + // the other must be init. + if !a_layout.is_uninit_valid() && !b_layout.is_uninit_valid() { + let (a, b) = + self.read_immediate(op, ExpectedKind::InitScalar)?.to_scalar_pair(); + self.visit_scalar(a, a_layout)?; + self.visit_scalar(b, b_layout)?; + } + } + Abi::Vector { .. } => { + // No checks here, we assume layout computation gets this right. + // (This is harder to check since Miri does not represent these as `Immediate`. We + // also cannot use field projections since this might be a newtype around a vector.) + } + Abi::Aggregate { .. } => { + // Nothing to do. + } + } + Ok(()) } } diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs index 879ae198f7e55..4ec19d9e655b7 100644 --- a/compiler/rustc_const_eval/src/interpret/visitor.rs +++ b/compiler/rustc_const_eval/src/interpret/visitor.rs @@ -1,544 +1,204 @@ //! Visitor for a run-time value with a given layout: Traverse enums, structs and other compound //! types until we arrive at the leaves, with custom handling for primitive types. +use rustc_index::IndexVec; use rustc_middle::mir::interpret::InterpResult; use rustc_middle::ty; -use rustc_middle::ty::layout::TyAndLayout; +use rustc_target::abi::FieldIdx; use rustc_target::abi::{FieldsShape, VariantIdx, Variants}; use std::num::NonZeroUsize; -use super::{InterpCx, MPlaceTy, Machine, OpTy, PlaceTy}; +use super::{InterpCx, MPlaceTy, Machine, Projectable}; -/// A thing that we can project into, and that has a layout. -/// This wouldn't have to depend on `Machine` but with the current type inference, -/// that's just more convenient to work with (avoids repeating all the `Machine` bounds). -pub trait Value<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized { - /// Gets this value's layout. - fn layout(&self) -> TyAndLayout<'tcx>; +/// How to traverse a value and what to do when we are at the leaves. +pub trait ValueVisitor<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized { + type V: Projectable<'mir, 'tcx, M::Provenance> + + From> + + std::fmt::Debug; - /// Makes this into an `OpTy`, in a cheap way that is good for reading. - fn to_op_for_read( - &self, - ecx: &InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>>; - - /// Makes this into an `OpTy`, in a potentially more expensive way that is good for projections. - fn to_op_for_proj( - &self, - ecx: &InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - self.to_op_for_read(ecx) - } - - /// Creates this from an `OpTy`. - /// - /// If `to_op_for_proj` only ever produces `Indirect` operands, then this one is definitely `Indirect`. - fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self; - - /// Projects to the given enum variant. - fn project_downcast( - &self, - ecx: &InterpCx<'mir, 'tcx, M>, - variant: VariantIdx, - ) -> InterpResult<'tcx, Self>; - - /// Projects to the n-th field. - fn project_field( - &self, - ecx: &InterpCx<'mir, 'tcx, M>, - field: usize, - ) -> InterpResult<'tcx, Self>; -} - -/// A thing that we can project into given *mutable* access to `ecx`, and that has a layout. -/// This wouldn't have to depend on `Machine` but with the current type inference, -/// that's just more convenient to work with (avoids repeating all the `Machine` bounds). -pub trait ValueMut<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized { - /// Gets this value's layout. - fn layout(&self) -> TyAndLayout<'tcx>; - - /// Makes this into an `OpTy`, in a cheap way that is good for reading. - fn to_op_for_read( - &self, - ecx: &InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>>; - - /// Makes this into an `OpTy`, in a potentially more expensive way that is good for projections. - fn to_op_for_proj( - &self, - ecx: &mut InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>>; - - /// Creates this from an `OpTy`. - /// - /// If `to_op_for_proj` only ever produces `Indirect` operands, then this one is definitely `Indirect`. - fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self; - - /// Projects to the given enum variant. - fn project_downcast( - &self, - ecx: &mut InterpCx<'mir, 'tcx, M>, - variant: VariantIdx, - ) -> InterpResult<'tcx, Self>; - - /// Projects to the n-th field. - fn project_field( - &self, - ecx: &mut InterpCx<'mir, 'tcx, M>, - field: usize, - ) -> InterpResult<'tcx, Self>; -} - -// We cannot have a general impl which shows that Value implies ValueMut. (When we do, it says we -// cannot `impl ValueMut for PlaceTy` because some downstream crate could `impl Value for PlaceTy`.) -// So we have some copy-paste here. (We could have a macro but since we only have 2 types with this -// double-impl, that would barely make the code shorter, if at all.) - -impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::Provenance> { - #[inline(always)] - fn layout(&self) -> TyAndLayout<'tcx> { - self.layout - } - - #[inline(always)] - fn to_op_for_read( - &self, - _ecx: &InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - Ok(self.clone()) - } - - #[inline(always)] - fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self { - op.clone() - } - - #[inline(always)] - fn project_downcast( - &self, - ecx: &InterpCx<'mir, 'tcx, M>, - variant: VariantIdx, - ) -> InterpResult<'tcx, Self> { - ecx.operand_downcast(self, variant) - } - - #[inline(always)] - fn project_field( - &self, - ecx: &InterpCx<'mir, 'tcx, M>, - field: usize, - ) -> InterpResult<'tcx, Self> { - ecx.operand_field(self, field) - } -} - -impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M> - for OpTy<'tcx, M::Provenance> -{ - #[inline(always)] - fn layout(&self) -> TyAndLayout<'tcx> { - self.layout - } - - #[inline(always)] - fn to_op_for_read( - &self, - _ecx: &InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - Ok(self.clone()) - } - - #[inline(always)] - fn to_op_for_proj( - &self, - _ecx: &mut InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - Ok(self.clone()) - } - - #[inline(always)] - fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self { - op.clone() - } - - #[inline(always)] - fn project_downcast( - &self, - ecx: &mut InterpCx<'mir, 'tcx, M>, - variant: VariantIdx, - ) -> InterpResult<'tcx, Self> { - ecx.operand_downcast(self, variant) - } - - #[inline(always)] - fn project_field( - &self, - ecx: &mut InterpCx<'mir, 'tcx, M>, - field: usize, - ) -> InterpResult<'tcx, Self> { - ecx.operand_field(self, field) - } -} - -impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> - for MPlaceTy<'tcx, M::Provenance> -{ - #[inline(always)] - fn layout(&self) -> TyAndLayout<'tcx> { - self.layout - } - - #[inline(always)] - fn to_op_for_read( - &self, - _ecx: &InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - Ok(self.into()) - } - - #[inline(always)] - fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self { - // assert is justified because our `to_op_for_read` only ever produces `Indirect` operands. - op.assert_mem_place() - } - - #[inline(always)] - fn project_downcast( - &self, - ecx: &InterpCx<'mir, 'tcx, M>, - variant: VariantIdx, - ) -> InterpResult<'tcx, Self> { - ecx.mplace_downcast(self, variant) - } - - #[inline(always)] - fn project_field( - &self, - ecx: &InterpCx<'mir, 'tcx, M>, - field: usize, - ) -> InterpResult<'tcx, Self> { - ecx.mplace_field(self, field) - } -} - -impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M> - for MPlaceTy<'tcx, M::Provenance> -{ - #[inline(always)] - fn layout(&self) -> TyAndLayout<'tcx> { - self.layout - } - - #[inline(always)] - fn to_op_for_read( - &self, - _ecx: &InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - Ok(self.into()) - } - - #[inline(always)] - fn to_op_for_proj( - &self, - _ecx: &mut InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - Ok(self.into()) - } - - #[inline(always)] - fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self { - // assert is justified because our `to_op_for_proj` only ever produces `Indirect` operands. - op.assert_mem_place() - } - - #[inline(always)] - fn project_downcast( - &self, - ecx: &mut InterpCx<'mir, 'tcx, M>, - variant: VariantIdx, - ) -> InterpResult<'tcx, Self> { - ecx.mplace_downcast(self, variant) - } - - #[inline(always)] - fn project_field( - &self, - ecx: &mut InterpCx<'mir, 'tcx, M>, - field: usize, - ) -> InterpResult<'tcx, Self> { - ecx.mplace_field(self, field) - } -} - -impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M> - for PlaceTy<'tcx, M::Provenance> -{ - #[inline(always)] - fn layout(&self) -> TyAndLayout<'tcx> { - self.layout - } - - #[inline(always)] - fn to_op_for_read( - &self, - ecx: &InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - // No need for `force_allocation` since we are just going to read from this. - ecx.place_to_op(self) - } + /// The visitor must have an `InterpCx` in it. + fn ecx(&self) -> &InterpCx<'mir, 'tcx, M>; + /// `read_discriminant` can be hooked for better error messages. #[inline(always)] - fn to_op_for_proj( - &self, - ecx: &mut InterpCx<'mir, 'tcx, M>, - ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - // We `force_allocation` here so that `from_op` below can work. - Ok(ecx.force_allocation(self)?.into()) + fn read_discriminant(&mut self, v: &Self::V) -> InterpResult<'tcx, VariantIdx> { + Ok(self.ecx().read_discriminant(&v.to_op(self.ecx())?)?) } - #[inline(always)] - fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self { - // assert is justified because our `to_op` only ever produces `Indirect` operands. - op.assert_mem_place().into() - } - - #[inline(always)] - fn project_downcast( - &self, - ecx: &mut InterpCx<'mir, 'tcx, M>, - variant: VariantIdx, - ) -> InterpResult<'tcx, Self> { - ecx.place_downcast(self, variant) - } - - #[inline(always)] - fn project_field( - &self, - ecx: &mut InterpCx<'mir, 'tcx, M>, - field: usize, - ) -> InterpResult<'tcx, Self> { - ecx.place_field(self, field) - } -} - -macro_rules! make_value_visitor { - ($visitor_trait:ident, $value_trait:ident, $($mutability:ident)?) => { - /// How to traverse a value and what to do when we are at the leaves. - pub trait $visitor_trait<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized { - type V: $value_trait<'mir, 'tcx, M>; - - /// The visitor must have an `InterpCx` in it. - fn ecx(&$($mutability)? self) - -> &$($mutability)? InterpCx<'mir, 'tcx, M>; - - /// `read_discriminant` can be hooked for better error messages. - #[inline(always)] - fn read_discriminant( - &mut self, - op: &OpTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, VariantIdx> { - Ok(self.ecx().read_discriminant(op)?.1) - } - - // Recursive actions, ready to be overloaded. - /// Visits the given value, dispatching as appropriate to more specialized visitors. - #[inline(always)] - fn visit_value(&mut self, v: &Self::V) -> InterpResult<'tcx> - { - self.walk_value(v) - } - /// Visits the given value as a union. No automatic recursion can happen here. - #[inline(always)] - fn visit_union(&mut self, _v: &Self::V, _fields: NonZeroUsize) -> InterpResult<'tcx> - { - Ok(()) - } - /// Visits the given value as the pointer of a `Box`. There is nothing to recurse into. - /// The type of `v` will be a raw pointer, but this is a field of `Box` and the - /// pointee type is the actual `T`. - #[inline(always)] - fn visit_box(&mut self, _v: &Self::V) -> InterpResult<'tcx> - { - Ok(()) + /// This function provides the chance to reorder the order in which fields are visited for + /// `FieldsShape::Aggregate`: The order of fields will be + /// `(0..num_fields).map(aggregate_field_order)`. + /// + /// The default means we iterate in source declaration order; alternative this can do an inverse + /// lookup in `memory_index` to use memory field order instead. + #[inline(always)] + fn aggregate_field_order(_memory_index: &IndexVec, idx: usize) -> usize { + idx + } + + // Recursive actions, ready to be overloaded. + /// Visits the given value, dispatching as appropriate to more specialized visitors. + #[inline(always)] + fn visit_value(&mut self, v: &Self::V) -> InterpResult<'tcx> { + self.walk_value(v) + } + /// Visits the given value as a union. No automatic recursion can happen here. + #[inline(always)] + fn visit_union(&mut self, _v: &Self::V, _fields: NonZeroUsize) -> InterpResult<'tcx> { + Ok(()) + } + /// Visits the given value as the pointer of a `Box`. There is nothing to recurse into. + /// The type of `v` will be a raw pointer, but this is a field of `Box` and the + /// pointee type is the actual `T`. + #[inline(always)] + fn visit_box(&mut self, _v: &Self::V) -> InterpResult<'tcx> { + Ok(()) + } + + /// Called each time we recurse down to a field of a "product-like" aggregate + /// (structs, tuples, arrays and the like, but not enums), passing in old (outer) + /// and new (inner) value. + /// This gives the visitor the chance to track the stack of nested fields that + /// we are descending through. + #[inline(always)] + fn visit_field( + &mut self, + _old_val: &Self::V, + _field: usize, + new_val: &Self::V, + ) -> InterpResult<'tcx> { + self.visit_value(new_val) + } + /// Called when recursing into an enum variant. + /// This gives the visitor the chance to track the stack of nested fields that + /// we are descending through. + #[inline(always)] + fn visit_variant( + &mut self, + _old_val: &Self::V, + _variant: VariantIdx, + new_val: &Self::V, + ) -> InterpResult<'tcx> { + self.visit_value(new_val) + } + + fn walk_value(&mut self, v: &Self::V) -> InterpResult<'tcx> { + let ty = v.layout().ty; + trace!("walk_value: type: {ty}"); + + // Special treatment for special types, where the (static) layout is not sufficient. + match *ty.kind() { + // If it is a trait object, switch to the real type that was used to create it. + ty::Dynamic(_, _, ty::Dyn) => { + // Dyn types. This is unsized, and the actual dynamic type of the data is given by the + // vtable stored in the place metadata. + // unsized values are never immediate, so we can assert_mem_place + let op = v.to_op(self.ecx())?; + let dest = op.assert_mem_place(); + let inner_mplace = self.ecx().unpack_dyn_trait(&dest)?.0; + trace!("walk_value: dyn object layout: {:#?}", inner_mplace.layout); + // recurse with the inner type + return self.visit_field(&v, 0, &inner_mplace.into()); } - /// Visits this value as an aggregate, you are getting an iterator yielding - /// all the fields (still in an `InterpResult`, you have to do error handling yourself). - /// Recurses into the fields. - #[inline(always)] - fn visit_aggregate( - &mut self, - v: &Self::V, - fields: impl Iterator>, - ) -> InterpResult<'tcx> { - self.walk_aggregate(v, fields) + ty::Dynamic(_, _, ty::DynStar) => { + // DynStar types. Very different from a dyn type (but strangely part of the + // same variant in `TyKind`): These are pairs where the 2nd component is the + // vtable, and the first component is the data (which must be ptr-sized). + let data = self.ecx().unpack_dyn_star(v)?.0; + return self.visit_field(&v, 0, &data); } - - /// Called each time we recurse down to a field of a "product-like" aggregate - /// (structs, tuples, arrays and the like, but not enums), passing in old (outer) - /// and new (inner) value. - /// This gives the visitor the chance to track the stack of nested fields that - /// we are descending through. - #[inline(always)] - fn visit_field( - &mut self, - _old_val: &Self::V, - _field: usize, - new_val: &Self::V, - ) -> InterpResult<'tcx> { - self.visit_value(new_val) + // Slices do not need special handling here: they have `Array` field + // placement with length 0, so we enter the `Array` case below which + // indirectly uses the metadata to determine the actual length. + + // However, `Box`... let's talk about `Box`. + ty::Adt(def, ..) if def.is_box() => { + // `Box` is a hybrid primitive-library-defined type that one the one hand is + // a dereferenceable pointer, on the other hand has *basically arbitrary + // user-defined layout* since the user controls the 'allocator' field. So it + // cannot be treated like a normal pointer, since it does not fit into an + // `Immediate`. Yeah, it is quite terrible. But many visitors want to do + // something with "all boxed pointers", so we handle this mess for them. + // + // When we hit a `Box`, we do not do the usual field recursion; instead, + // we (a) call `visit_box` on the pointer value, and (b) recurse on the + // allocator field. We also assert tons of things to ensure we do not miss + // any other fields. + + // `Box` has two fields: the pointer we care about, and the allocator. + assert_eq!(v.layout().fields.count(), 2, "`Box` must have exactly 2 fields"); + let (unique_ptr, alloc) = + (self.ecx().project_field(v, 0)?, self.ecx().project_field(v, 1)?); + // Unfortunately there is some type junk in the way here: `unique_ptr` is a `Unique`... + // (which means another 2 fields, the second of which is a `PhantomData`) + assert_eq!(unique_ptr.layout().fields.count(), 2); + let (nonnull_ptr, phantom) = ( + self.ecx().project_field(&unique_ptr, 0)?, + self.ecx().project_field(&unique_ptr, 1)?, + ); + assert!( + phantom.layout().ty.ty_adt_def().is_some_and(|adt| adt.is_phantom_data()), + "2nd field of `Unique` should be PhantomData but is {:?}", + phantom.layout().ty, + ); + // ... that contains a `NonNull`... (gladly, only a single field here) + assert_eq!(nonnull_ptr.layout().fields.count(), 1); + let raw_ptr = self.ecx().project_field(&nonnull_ptr, 0)?; // the actual raw ptr + // ... whose only field finally is a raw ptr we can dereference. + self.visit_box(&raw_ptr)?; + + // The second `Box` field is the allocator, which we recursively check for validity + // like in regular structs. + self.visit_field(v, 1, &alloc)?; + + // We visited all parts of this one. + return Ok(()); } - /// Called when recursing into an enum variant. - /// This gives the visitor the chance to track the stack of nested fields that - /// we are descending through. - #[inline(always)] - fn visit_variant( - &mut self, - _old_val: &Self::V, - _variant: VariantIdx, - new_val: &Self::V, - ) -> InterpResult<'tcx> { - self.visit_value(new_val) + _ => {} + }; + + // Visit the fields of this value. + match &v.layout().fields { + FieldsShape::Primitive => {} + &FieldsShape::Union(fields) => { + self.visit_union(v, fields)?; } - - // Default recursors. Not meant to be overloaded. - fn walk_aggregate( - &mut self, - v: &Self::V, - fields: impl Iterator>, - ) -> InterpResult<'tcx> { - // Now iterate over it. - for (idx, field_val) in fields.enumerate() { - self.visit_field(v, idx, &field_val?)?; + FieldsShape::Arbitrary { offsets, memory_index } => { + for idx in 0..offsets.len() { + let idx = Self::aggregate_field_order(memory_index, idx); + let field = self.ecx().project_field(v, idx)?; + self.visit_field(v, idx, &field)?; } - Ok(()) } - fn walk_value(&mut self, v: &Self::V) -> InterpResult<'tcx> - { - let ty = v.layout().ty; - trace!("walk_value: type: {ty}"); - - // Special treatment for special types, where the (static) layout is not sufficient. - match *ty.kind() { - // If it is a trait object, switch to the real type that was used to create it. - ty::Dynamic(_, _, ty::Dyn) => { - // Dyn types. This is unsized, and the actual dynamic type of the data is given by the - // vtable stored in the place metadata. - // unsized values are never immediate, so we can assert_mem_place - let op = v.to_op_for_read(self.ecx())?; - let dest = op.assert_mem_place(); - let inner_mplace = self.ecx().unpack_dyn_trait(&dest)?.0; - trace!("walk_value: dyn object layout: {:#?}", inner_mplace.layout); - // recurse with the inner type - return self.visit_field(&v, 0, &$value_trait::from_op(&inner_mplace.into())); - }, - ty::Dynamic(_, _, ty::DynStar) => { - // DynStar types. Very different from a dyn type (but strangely part of the - // same variant in `TyKind`): These are pairs where the 2nd component is the - // vtable, and the first component is the data (which must be ptr-sized). - let op = v.to_op_for_proj(self.ecx())?; - let data = self.ecx().unpack_dyn_star(&op)?.0; - return self.visit_field(&v, 0, &$value_trait::from_op(&data)); - } - // Slices do not need special handling here: they have `Array` field - // placement with length 0, so we enter the `Array` case below which - // indirectly uses the metadata to determine the actual length. - - // However, `Box`... let's talk about `Box`. - ty::Adt(def, ..) if def.is_box() => { - // `Box` is a hybrid primitive-library-defined type that one the one hand is - // a dereferenceable pointer, on the other hand has *basically arbitrary - // user-defined layout* since the user controls the 'allocator' field. So it - // cannot be treated like a normal pointer, since it does not fit into an - // `Immediate`. Yeah, it is quite terrible. But many visitors want to do - // something with "all boxed pointers", so we handle this mess for them. - // - // When we hit a `Box`, we do not do the usual `visit_aggregate`; instead, - // we (a) call `visit_box` on the pointer value, and (b) recurse on the - // allocator field. We also assert tons of things to ensure we do not miss - // any other fields. - - // `Box` has two fields: the pointer we care about, and the allocator. - assert_eq!(v.layout().fields.count(), 2, "`Box` must have exactly 2 fields"); - let (unique_ptr, alloc) = - (v.project_field(self.ecx(), 0)?, v.project_field(self.ecx(), 1)?); - // Unfortunately there is some type junk in the way here: `unique_ptr` is a `Unique`... - // (which means another 2 fields, the second of which is a `PhantomData`) - assert_eq!(unique_ptr.layout().fields.count(), 2); - let (nonnull_ptr, phantom) = ( - unique_ptr.project_field(self.ecx(), 0)?, - unique_ptr.project_field(self.ecx(), 1)?, - ); - assert!( - phantom.layout().ty.ty_adt_def().is_some_and(|adt| adt.is_phantom_data()), - "2nd field of `Unique` should be PhantomData but is {:?}", - phantom.layout().ty, - ); - // ... that contains a `NonNull`... (gladly, only a single field here) - assert_eq!(nonnull_ptr.layout().fields.count(), 1); - let raw_ptr = nonnull_ptr.project_field(self.ecx(), 0)?; // the actual raw ptr - // ... whose only field finally is a raw ptr we can dereference. - self.visit_box(&raw_ptr)?; - - // The second `Box` field is the allocator, which we recursively check for validity - // like in regular structs. - self.visit_field(v, 1, &alloc)?; - - // We visited all parts of this one. - return Ok(()); - } - _ => {}, - }; - - // Visit the fields of this value. - match &v.layout().fields { - FieldsShape::Primitive => {} - &FieldsShape::Union(fields) => { - self.visit_union(v, fields)?; - } - FieldsShape::Arbitrary { offsets, .. } => { - // FIXME: We collect in a vec because otherwise there are lifetime - // errors: Projecting to a field needs access to `ecx`. - let fields: Vec> = - (0..offsets.len()).map(|i| { - v.project_field(self.ecx(), i) - }) - .collect(); - self.visit_aggregate(v, fields.into_iter())?; - } - FieldsShape::Array { .. } => { - // Let's get an mplace (or immediate) first. - // This might `force_allocate` if `v` is a `PlaceTy`, but `place_index` does that anyway. - let op = v.to_op_for_proj(self.ecx())?; - // Now we can go over all the fields. - // This uses the *run-time length*, i.e., if we are a slice, - // the dynamic info from the metadata is used. - let iter = self.ecx().operand_array_fields(&op)? - .map(|f| f.and_then(|f| { - Ok($value_trait::from_op(&f)) - })); - self.visit_aggregate(v, iter)?; - } + FieldsShape::Array { .. } => { + for (idx, field) in self.ecx().project_array_fields(v)?.enumerate() { + self.visit_field(v, idx, &field?)?; } + } + } - match v.layout().variants { - // If this is a multi-variant layout, find the right variant and proceed - // with *its* fields. - Variants::Multiple { .. } => { - let op = v.to_op_for_read(self.ecx())?; - let idx = self.read_discriminant(&op)?; - let inner = v.project_downcast(self.ecx(), idx)?; - trace!("walk_value: variant layout: {:#?}", inner.layout()); - // recurse with the inner type - self.visit_variant(v, idx, &inner) - } - // For single-variant layouts, we already did anything there is to do. - Variants::Single { .. } => Ok(()) - } + match v.layout().variants { + // If this is a multi-variant layout, find the right variant and proceed + // with *its* fields. + Variants::Multiple { .. } => { + let idx = self.read_discriminant(v)?; + // There are 3 cases where downcasts can turn a Scalar/ScalarPair into a different ABI which + // could be a problem for `ImmTy` (see layout_sanity_check): + // - variant.size == Size::ZERO: works fine because `ImmTy::offset` has a special case for + // zero-sized layouts. + // - variant.fields.count() == 0: works fine because `ImmTy::offset` has a special case for + // zero-field aggregates. + // - variant.abi.is_uninhabited(): triggers UB in `read_discriminant` so we never get here. + let inner = self.ecx().project_downcast(v, idx)?; + trace!("walk_value: variant layout: {:#?}", inner.layout()); + // recurse with the inner type + self.visit_variant(v, idx, &inner)?; } + // For single-variant layouts, we already did anything there is to do. + Variants::Single { .. } => {} } + + Ok(()) } } - -make_value_visitor!(ValueVisitor, Value,); -make_value_visitor!(MutValueVisitor, ValueMut, mut); diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs index b8030d9db13c5..c1cb2f2e497f4 100644 --- a/compiler/rustc_middle/src/mir/interpret/allocation.rs +++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs @@ -571,7 +571,7 @@ impl Allocation assert!(self.mutability == Mutability::Mut); // `to_bits_or_ptr_internal` is the right method because we just want to store this data - // as-is into memory. + // as-is into memory. This also double-checks that `val.size()` matches `range.size`. let (bytes, provenance) = match val.to_bits_or_ptr_internal(range.size)? { Right(ptr) => { let (provenance, offset) = ptr.into_parts(); diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs index 372452ea29a8d..690fcfe1be409 100644 --- a/compiler/rustc_middle/src/mir/interpret/error.rs +++ b/compiler/rustc_middle/src/mir/interpret/error.rs @@ -12,7 +12,8 @@ use rustc_errors::{ use rustc_macros::HashStable; use rustc_session::CtfeBacktrace; use rustc_span::def_id::DefId; -use rustc_target::abi::{call, Align, Size, WrappingRange}; +use rustc_target::abi::{call, Align, Size, VariantIdx, WrappingRange}; + use std::borrow::Cow; use std::{any::Any, backtrace::Backtrace, fmt}; @@ -191,9 +192,8 @@ pub enum InvalidProgramInfo<'tcx> { FnAbiAdjustForForeignAbi(call::AdjustForForeignAbiError), /// SizeOf of unsized type was requested. SizeOfUnsizedType(Ty<'tcx>), - /// An unsized local was accessed without having been initialized. - /// This is not meaningful as we can't even have backing memory for such locals. - UninitUnsizedLocal, + /// We are runnning into a nonsense situation due to ConstProp violating our invariants. + ConstPropNonsense, } /// Details of why a pointer had to be in-bounds. @@ -324,7 +324,9 @@ pub enum UndefinedBehaviorInfo<'a> { /// Data size is not equal to target size. ScalarSizeMismatch(ScalarSizeMismatch), /// A discriminant of an uninhabited enum variant is written. - UninhabitedEnumVariantWritten, + UninhabitedEnumVariantWritten(VariantIdx), + /// An uninhabited enum variant is projected. + UninhabitedEnumVariantRead(VariantIdx), /// Validation error. Validation(ValidationErrorInfo<'a>), // FIXME(fee1-dead) these should all be actual variants of the enum instead of dynamically @@ -394,6 +396,7 @@ pub enum ValidationErrorKind<'tcx> { UnsafeCell, UninhabitedVal { ty: Ty<'tcx> }, InvalidEnumTag { value: String }, + UninhabitedEnumTag, UninitEnumTag, UninitStr, Uninit { expected: ExpectedKind }, diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs index 0416411dfe14a..47421d0f03719 100644 --- a/compiler/rustc_middle/src/mir/interpret/value.rs +++ b/compiler/rustc_middle/src/mir/interpret/value.rs @@ -320,6 +320,14 @@ impl Scalar { } }) } + + #[inline] + pub fn size(self) -> Size { + match self { + Scalar::Int(int) => int.size(), + Scalar::Ptr(_ptr, sz) => Size::from_bytes(sz), + } + } } impl<'tcx, Prov: Provenance> Scalar { diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index 62805d1e8b5c9..81e7dc3728ab0 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -741,9 +741,9 @@ where let fields = match this.ty.kind() { ty::Adt(def, _) if def.variants().is_empty() => - bug!("for_variant called on zero-variant enum"), + bug!("for_variant called on zero-variant enum {}", this.ty), ty::Adt(def, _) => def.variant(variant_index).fields.len(), - _ => bug!(), + _ => bug!("`ty_and_layout_for_variant` on unexpected type {}", this.ty), }; tcx.mk_layout(LayoutS { variants: Variants::Single { index: variant_index }, diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index f9c1ca9a8b167..3e023ccdead67 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -2670,11 +2670,6 @@ impl<'tcx> Ty<'tcx> { variant_index: VariantIdx, ) -> Option> { match self.kind() { - TyKind::Adt(adt, _) if adt.variants().is_empty() => { - // This can actually happen during CTFE, see - // https://github.com/rust-lang/rust/issues/89765. - None - } TyKind::Adt(adt, _) if adt.is_enum() => { Some(adt.discriminant_for_variant(tcx, variant_index)) } diff --git a/src/tools/clippy/tests/ui/self_assignment.rs b/src/tools/clippy/tests/ui/self_assignment.rs index d6682cc63dcf9..a7f9fbaae7cf6 100644 --- a/src/tools/clippy/tests/ui/self_assignment.rs +++ b/src/tools/clippy/tests/ui/self_assignment.rs @@ -14,7 +14,7 @@ pub fn positives(mut a: usize, b: &mut u32, mut s: S) { *b = *b; s = s; s.a = s.a; - s.b[10] = s.b[5 + 5]; + s.b[9] = s.b[5 + 4]; s.c[0][1] = s.c[0][1]; s.b[a] = s.b[a]; *s.e = *s.e; diff --git a/src/tools/clippy/tests/ui/self_assignment.stderr b/src/tools/clippy/tests/ui/self_assignment.stderr index bed88244eea70..25b8569fa3d0a 100644 --- a/src/tools/clippy/tests/ui/self_assignment.stderr +++ b/src/tools/clippy/tests/ui/self_assignment.stderr @@ -24,11 +24,11 @@ error: self-assignment of `s.a` to `s.a` LL | s.a = s.a; | ^^^^^^^^^ -error: self-assignment of `s.b[5 + 5]` to `s.b[10]` +error: self-assignment of `s.b[5 + 4]` to `s.b[9]` --> $DIR/self_assignment.rs:17:5 | -LL | s.b[10] = s.b[5 + 5]; - | ^^^^^^^^^^^^^^^^^^^^ +LL | s.b[9] = s.b[5 + 4]; + | ^^^^^^^^^^^^^^^^^^^ error: self-assignment of `s.c[0][1]` to `s.c[0][1]` --> $DIR/self_assignment.rs:18:5 diff --git a/src/tools/miri/src/borrow_tracker/stacked_borrows/mod.rs b/src/tools/miri/src/borrow_tracker/stacked_borrows/mod.rs index 1aed436e88d82..8059fddb1008a 100644 --- a/src/tools/miri/src/borrow_tracker/stacked_borrows/mod.rs +++ b/src/tools/miri/src/borrow_tracker/stacked_borrows/mod.rs @@ -928,13 +928,13 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { Ok(()) } } - impl<'ecx, 'mir, 'tcx> MutValueVisitor<'mir, 'tcx, MiriMachine<'mir, 'tcx>> + impl<'ecx, 'mir, 'tcx> ValueVisitor<'mir, 'tcx, MiriMachine<'mir, 'tcx>> for RetagVisitor<'ecx, 'mir, 'tcx> { type V = PlaceTy<'tcx, Provenance>; #[inline(always)] - fn ecx(&mut self) -> &mut MiriInterpCx<'mir, 'tcx> { + fn ecx(&self) -> &MiriInterpCx<'mir, 'tcx> { self.ecx } diff --git a/src/tools/miri/src/borrow_tracker/tree_borrows/mod.rs b/src/tools/miri/src/borrow_tracker/tree_borrows/mod.rs index 2afd45829bd89..9073f6954429a 100644 --- a/src/tools/miri/src/borrow_tracker/tree_borrows/mod.rs +++ b/src/tools/miri/src/borrow_tracker/tree_borrows/mod.rs @@ -413,13 +413,13 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { Ok(()) } } - impl<'ecx, 'mir, 'tcx> MutValueVisitor<'mir, 'tcx, MiriMachine<'mir, 'tcx>> + impl<'ecx, 'mir, 'tcx> ValueVisitor<'mir, 'tcx, MiriMachine<'mir, 'tcx>> for RetagVisitor<'ecx, 'mir, 'tcx> { type V = PlaceTy<'tcx, Provenance>; #[inline(always)] - fn ecx(&mut self) -> &mut MiriInterpCx<'mir, 'tcx> { + fn ecx(&self) -> &MiriInterpCx<'mir, 'tcx> { self.ecx } @@ -578,14 +578,14 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { /// I.e. input is what you get from the visitor upon encountering an `adt` that is `Unique`, /// and output can be used by `retag_ptr_inplace`. fn inner_ptr_of_unique<'tcx>( - ecx: &mut MiriInterpCx<'_, 'tcx>, + ecx: &MiriInterpCx<'_, 'tcx>, place: &PlaceTy<'tcx, Provenance>, ) -> InterpResult<'tcx, PlaceTy<'tcx, Provenance>> { // Follows the same layout as `interpret/visitor.rs:walk_value` for `Box` in // `rustc_const_eval`, just with one fewer layer. // Here we have a `Unique(NonNull(*mut), PhantomData)` assert_eq!(place.layout.fields.count(), 2, "Unique must have exactly 2 fields"); - let (nonnull, phantom) = (ecx.place_field(place, 0)?, ecx.place_field(place, 1)?); + let (nonnull, phantom) = (ecx.project_field(place, 0)?, ecx.project_field(place, 1)?); assert!( phantom.layout.ty.ty_adt_def().is_some_and(|adt| adt.is_phantom_data()), "2nd field of `Unique` should be `PhantomData` but is `{:?}`", @@ -593,7 +593,7 @@ fn inner_ptr_of_unique<'tcx>( ); // Now down to `NonNull(*mut)` assert_eq!(nonnull.layout.fields.count(), 1, "NonNull must have exactly 1 field"); - let ptr = ecx.place_field(&nonnull, 0)?; + let ptr = ecx.project_field(&nonnull, 0)?; // Finally a plain `*mut` Ok(ptr) } diff --git a/src/tools/miri/src/eval.rs b/src/tools/miri/src/eval.rs index 79a7115f24e6d..4f7b70649a934 100644 --- a/src/tools/miri/src/eval.rs +++ b/src/tools/miri/src/eval.rs @@ -320,7 +320,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>( ))?; let argvs_place = ecx.allocate(argvs_layout, MiriMemoryKind::Machine.into())?; for (idx, arg) in argvs.into_iter().enumerate() { - let place = ecx.mplace_field(&argvs_place, idx)?; + let place = ecx.project_field(&argvs_place, idx)?; ecx.write_immediate(arg, &place.into())?; } ecx.mark_immutable(&argvs_place); @@ -354,7 +354,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>( ecx.machine.cmd_line = Some(*cmd_place); // Store the UTF-16 string. We just allocated so we know the bounds are fine. for (idx, &c) in cmd_utf16.iter().enumerate() { - let place = ecx.mplace_field(&cmd_place, idx)?; + let place = ecx.project_field(&cmd_place, idx)?; ecx.write_scalar(Scalar::from_u16(c), &place.into())?; } ecx.mark_immutable(&cmd_place); diff --git a/src/tools/miri/src/helpers.rs b/src/tools/miri/src/helpers.rs index c19d691ede6ad..f6a438f5d629b 100644 --- a/src/tools/miri/src/helpers.rs +++ b/src/tools/miri/src/helpers.rs @@ -10,6 +10,7 @@ use log::trace; use rustc_hir::def::{DefKind, Namespace}; use rustc_hir::def_id::{DefId, CRATE_DEF_INDEX}; +use rustc_index::IndexVec; use rustc_middle::mir; use rustc_middle::ty::{ self, @@ -17,7 +18,7 @@ use rustc_middle::ty::{ List, TyCtxt, }; use rustc_span::{def_id::CrateNum, sym, Span, Symbol}; -use rustc_target::abi::{Align, FieldsShape, Size, Variants}; +use rustc_target::abi::{Align, FieldIdx, FieldsShape, Size, Variants}; use rustc_target::spec::abi::Abi; use rand::RngCore; @@ -229,20 +230,20 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { this.layout_of(ty).unwrap() } - /// Project to the given *named* field of the mplace (which must be a struct or union type). - fn mplace_field_named( + /// Project to the given *named* field (which must be a struct or union type). + fn project_field_named>( &self, - mplace: &MPlaceTy<'tcx, Provenance>, + base: &P, name: &str, - ) -> InterpResult<'tcx, MPlaceTy<'tcx, Provenance>> { + ) -> InterpResult<'tcx, P> { let this = self.eval_context_ref(); - let adt = mplace.layout.ty.ty_adt_def().unwrap(); + let adt = base.layout().ty.ty_adt_def().unwrap(); for (idx, field) in adt.non_enum_variant().fields.iter().enumerate() { if field.name.as_str() == name { - return this.mplace_field(mplace, idx); + return this.project_field(base, idx); } } - bug!("No field named {} in type {}", name, mplace.layout.ty); + bug!("No field named {} in type {}", name, base.layout().ty); } /// Write an int of the appropriate size to `dest`. The target type may be signed or unsigned, @@ -270,7 +271,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { ) -> InterpResult<'tcx> { let this = self.eval_context_mut(); for (idx, &val) in values.iter().enumerate() { - let field = this.mplace_field(dest, idx)?; + let field = this.project_field(dest, idx)?; this.write_int(val, &field.into())?; } Ok(()) @@ -284,7 +285,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { ) -> InterpResult<'tcx> { let this = self.eval_context_mut(); for &(name, val) in values.iter() { - let field = this.mplace_field_named(dest, name)?; + let field = this.project_field_named(dest, name)?; this.write_int(val, &field.into())?; } Ok(()) @@ -301,8 +302,8 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } /// Get the `Place` for a local - fn local_place(&mut self, local: mir::Local) -> InterpResult<'tcx, PlaceTy<'tcx, Provenance>> { - let this = self.eval_context_mut(); + fn local_place(&self, local: mir::Local) -> InterpResult<'tcx, PlaceTy<'tcx, Provenance>> { + let this = self.eval_context_ref(); let place = mir::Place { local, projection: List::empty() }; this.eval_place(place) } @@ -479,6 +480,16 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { self.ecx } + fn aggregate_field_order(memory_index: &IndexVec, idx: usize) -> usize { + // We need to do an *inverse* lookup: find the field that has position `idx` in memory order. + for (src_field, &mem_pos) in memory_index.iter_enumerated() { + if mem_pos as usize == idx { + return src_field.as_usize(); + } + } + panic!("invalid `memory_index`, could not find {}-th field in memory order", idx); + } + // Hook to detect `UnsafeCell`. fn visit_value(&mut self, v: &MPlaceTy<'tcx, Provenance>) -> InterpResult<'tcx> { trace!("UnsafeCellVisitor: {:?} {:?}", *v, v.layout.ty); @@ -524,33 +535,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } } - // Make sure we visit aggregates in increasing offset order. - fn visit_aggregate( - &mut self, - place: &MPlaceTy<'tcx, Provenance>, - fields: impl Iterator>>, - ) -> InterpResult<'tcx> { - match place.layout.fields { - FieldsShape::Array { .. } => { - // For the array layout, we know the iterator will yield sorted elements so - // we can avoid the allocation. - self.walk_aggregate(place, fields) - } - FieldsShape::Arbitrary { .. } => { - // Gather the subplaces and sort them before visiting. - let mut places = fields - .collect::>>>()?; - // we just compare offsets, the abs. value never matters - places.sort_by_key(|place| place.ptr.addr()); - self.walk_aggregate(place, places.into_iter().map(Ok)) - } - FieldsShape::Union { .. } | FieldsShape::Primitive => { - // Uh, what? - bug!("unions/primitives are not aggregates we should ever visit") - } - } - } - fn visit_union( &mut self, _v: &MPlaceTy<'tcx, Provenance>, @@ -746,7 +730,8 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { Ok(mplace) } - fn deref_pointer_as( + /// Deref' a pointer *without* checking that the place is dereferenceable. + fn deref_pointer_unchecked( &self, val: &ImmTy<'tcx, Provenance>, layout: TyAndLayout<'tcx>, @@ -811,10 +796,10 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { tp: &MPlaceTy<'tcx, Provenance>, ) -> InterpResult<'tcx, Option> { let this = self.eval_context_mut(); - let seconds_place = this.mplace_field(tp, 0)?; + let seconds_place = this.project_field(tp, 0)?; let seconds_scalar = this.read_scalar(&seconds_place.into())?; let seconds = seconds_scalar.to_target_isize(this)?; - let nanoseconds_place = this.mplace_field(tp, 1)?; + let nanoseconds_place = this.project_field(tp, 1)?; let nanoseconds_scalar = this.read_scalar(&nanoseconds_place.into())?; let nanoseconds = nanoseconds_scalar.to_target_isize(this)?; diff --git a/src/tools/miri/src/shims/backtrace.rs b/src/tools/miri/src/shims/backtrace.rs index adf9a35d5c3a3..e4aa7467cbe04 100644 --- a/src/tools/miri/src/shims/backtrace.rs +++ b/src/tools/miri/src/shims/backtrace.rs @@ -83,7 +83,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { // Write pointers into array for (i, ptr) in ptrs.into_iter().enumerate() { - let place = this.mplace_index(&alloc, i as u64)?; + let place = this.project_index(&alloc, i as u64)?; this.write_pointer(ptr, &place.into())?; } @@ -196,33 +196,33 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { this.write_immediate( name_alloc.to_ref(this), - &this.mplace_field(&dest, 0)?.into(), + &this.project_field(&dest, 0)?.into(), )?; this.write_immediate( filename_alloc.to_ref(this), - &this.mplace_field(&dest, 1)?.into(), + &this.project_field(&dest, 1)?.into(), )?; } 1 => { this.write_scalar( Scalar::from_target_usize(name.len().try_into().unwrap(), this), - &this.mplace_field(&dest, 0)?.into(), + &this.project_field(&dest, 0)?.into(), )?; this.write_scalar( Scalar::from_target_usize(filename.len().try_into().unwrap(), this), - &this.mplace_field(&dest, 1)?.into(), + &this.project_field(&dest, 1)?.into(), )?; } _ => throw_unsup_format!("unknown `miri_resolve_frame` flags {}", flags), } - this.write_scalar(Scalar::from_u32(lineno), &this.mplace_field(&dest, 2)?.into())?; - this.write_scalar(Scalar::from_u32(colno), &this.mplace_field(&dest, 3)?.into())?; + this.write_scalar(Scalar::from_u32(lineno), &this.project_field(&dest, 2)?.into())?; + this.write_scalar(Scalar::from_u32(colno), &this.project_field(&dest, 3)?.into())?; // Support a 4-field struct for now - this is deprecated // and slated for removal. if num_fields == 5 { - this.write_pointer(fn_ptr, &this.mplace_field(&dest, 4)?.into())?; + this.write_pointer(fn_ptr, &this.project_field(&dest, 4)?.into())?; } Ok(()) diff --git a/src/tools/miri/src/shims/env.rs b/src/tools/miri/src/shims/env.rs index 1dcb877a83f0d..f98fd0431ae63 100644 --- a/src/tools/miri/src/shims/env.rs +++ b/src/tools/miri/src/shims/env.rs @@ -456,7 +456,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { ))?; let vars_place = this.allocate(vars_layout, MiriMemoryKind::Runtime.into())?; for (idx, var) in vars.into_iter().enumerate() { - let place = this.mplace_field(&vars_place, idx)?; + let place = this.project_field(&vars_place, idx)?; this.write_pointer(var, &place.into())?; } this.write_pointer(vars_place.ptr, &this.machine.env_vars.environ.unwrap().into())?; diff --git a/src/tools/miri/src/shims/foreign_items.rs b/src/tools/miri/src/shims/foreign_items.rs index 72a4adba8d41a..c753eadbbad35 100644 --- a/src/tools/miri/src/shims/foreign_items.rs +++ b/src/tools/miri/src/shims/foreign_items.rs @@ -942,9 +942,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { #[allow(clippy::arithmetic_side_effects)] // it's a u128, we can shift by 64 let (c_out, sum) = ((wide_sum >> 64).truncate::(), wide_sum.truncate::()); - let c_out_field = this.place_field(dest, 0)?; + let c_out_field = this.project_field(dest, 0)?; this.write_scalar(Scalar::from_u8(c_out), &c_out_field)?; - let sum_field = this.place_field(dest, 1)?; + let sum_field = this.project_field(dest, 1)?; this.write_scalar(Scalar::from_u64(sum), &sum_field)?; } "llvm.x86.sse2.pause" diff --git a/src/tools/miri/src/shims/intrinsics/simd.rs b/src/tools/miri/src/shims/intrinsics/simd.rs index 94f8cfbfb1c3d..3afe5214f08b2 100644 --- a/src/tools/miri/src/shims/intrinsics/simd.rs +++ b/src/tools/miri/src/shims/intrinsics/simd.rs @@ -57,8 +57,8 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { }; for i in 0..dest_len { - let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; - let dest = this.mplace_index(&dest, i)?; + let op = this.read_immediate(&this.project_index(&op, i)?.into())?; + let dest = this.project_index(&dest, i)?; let val = match which { Op::MirOp(mir_op) => this.unary_op(mir_op, &op)?.to_scalar(), Op::Abs => { @@ -172,9 +172,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { }; for i in 0..dest_len { - let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?; - let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?; - let dest = this.mplace_index(&dest, i)?; + let left = this.read_immediate(&this.project_index(&left, i)?.into())?; + let right = this.read_immediate(&this.project_index(&right, i)?.into())?; + let dest = this.project_index(&dest, i)?; let val = match which { Op::MirOp(mir_op) => { let (val, overflowed, ty) = this.overflowing_binary_op(mir_op, &left, &right)?; @@ -232,10 +232,10 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { assert_eq!(dest_len, c_len); for i in 0..dest_len { - let a = this.read_scalar(&this.mplace_index(&a, i)?.into())?; - let b = this.read_scalar(&this.mplace_index(&b, i)?.into())?; - let c = this.read_scalar(&this.mplace_index(&c, i)?.into())?; - let dest = this.mplace_index(&dest, i)?; + let a = this.read_scalar(&this.project_index(&a, i)?.into())?; + let b = this.read_scalar(&this.project_index(&b, i)?.into())?; + let c = this.read_scalar(&this.project_index(&c, i)?.into())?; + let dest = this.project_index(&dest, i)?; // Works for f32 and f64. // FIXME: using host floats to work around https://github.com/rust-lang/miri/issues/2468. @@ -295,13 +295,13 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { }; // Initialize with first lane, then proceed with the rest. - let mut res = this.read_immediate(&this.mplace_index(&op, 0)?.into())?; + let mut res = this.read_immediate(&this.project_index(&op, 0)?.into())?; if matches!(which, Op::MirOpBool(_)) { // Convert to `bool` scalar. res = imm_from_bool(simd_element_to_bool(res)?); } for i in 1..op_len { - let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; + let op = this.read_immediate(&this.project_index(&op, i)?.into())?; res = match which { Op::MirOp(mir_op) => { this.binary_op(mir_op, &res, &op)? @@ -355,7 +355,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { let mut res = init; for i in 0..op_len { - let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; + let op = this.read_immediate(&this.project_index(&op, i)?.into())?; res = this.binary_op(mir_op, &res, &op)?; } this.write_immediate(*res, dest)?; @@ -372,10 +372,10 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { assert_eq!(dest_len, no_len); for i in 0..dest_len { - let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?; - let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?; - let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?; - let dest = this.mplace_index(&dest, i)?; + let mask = this.read_immediate(&this.project_index(&mask, i)?.into())?; + let yes = this.read_immediate(&this.project_index(&yes, i)?.into())?; + let no = this.read_immediate(&this.project_index(&no, i)?.into())?; + let dest = this.project_index(&dest, i)?; let val = if simd_element_to_bool(mask)? { yes } else { no }; this.write_immediate(*val, &dest.into())?; @@ -403,9 +403,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { & 1u64 .checked_shl(simd_bitmask_index(i, dest_len, this.data_layout().endian)) .unwrap(); - let yes = this.read_immediate(&this.mplace_index(&yes, i.into())?.into())?; - let no = this.read_immediate(&this.mplace_index(&no, i.into())?.into())?; - let dest = this.mplace_index(&dest, i.into())?; + let yes = this.read_immediate(&this.project_index(&yes, i.into())?.into())?; + let no = this.read_immediate(&this.project_index(&no, i.into())?.into())?; + let dest = this.project_index(&dest, i.into())?; let val = if mask != 0 { yes } else { no }; this.write_immediate(*val, &dest.into())?; @@ -435,8 +435,8 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { let from_exposed_cast = intrinsic_name == "from_exposed_addr"; for i in 0..dest_len { - let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; - let dest = this.mplace_index(&dest, i)?; + let op = this.read_immediate(&this.project_index(&op, i)?.into())?; + let dest = this.project_index(&dest, i)?; let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) { // Int-to-(int|float): always safe @@ -496,17 +496,17 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { for i in 0..dest_len { let src_index: u64 = this - .read_immediate(&this.operand_index(index, i)?)? + .read_immediate(&this.project_index(index, i)?)? .to_scalar() .to_u32()? .into(); - let dest = this.mplace_index(&dest, i)?; + let dest = this.project_index(&dest, i)?; let val = if src_index < left_len { - this.read_immediate(&this.mplace_index(&left, src_index)?.into())? + this.read_immediate(&this.project_index(&left, src_index)?.into())? } else if src_index < left_len.checked_add(right_len).unwrap() { let right_idx = src_index.checked_sub(left_len).unwrap(); - this.read_immediate(&this.mplace_index(&right, right_idx)?.into())? + this.read_immediate(&this.project_index(&right, right_idx)?.into())? } else { span_bug!( this.cur_span(), @@ -528,10 +528,10 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { assert_eq!(dest_len, mask_len); for i in 0..dest_len { - let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?; - let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?; - let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?; - let dest = this.mplace_index(&dest, i)?; + let passthru = this.read_immediate(&this.project_index(&passthru, i)?.into())?; + let ptr = this.read_immediate(&this.project_index(&ptrs, i)?.into())?; + let mask = this.read_immediate(&this.project_index(&mask, i)?.into())?; + let dest = this.project_index(&dest, i)?; let val = if simd_element_to_bool(mask)? { let place = this.deref_operand(&ptr.into())?; @@ -552,9 +552,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { assert_eq!(ptrs_len, mask_len); for i in 0..ptrs_len { - let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?; - let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?; - let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?; + let value = this.read_immediate(&this.project_index(&value, i)?.into())?; + let ptr = this.read_immediate(&this.project_index(&ptrs, i)?.into())?; + let mask = this.read_immediate(&this.project_index(&mask, i)?.into())?; if simd_element_to_bool(mask)? { let place = this.deref_operand(&ptr.into())?; @@ -578,7 +578,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { let mut res = 0u64; for i in 0..op_len { - let op = this.read_immediate(&this.mplace_index(&op, i.into())?.into())?; + let op = this.read_immediate(&this.project_index(&op, i.into())?.into())?; if simd_element_to_bool(op)? { res |= 1u64 .checked_shl(simd_bitmask_index(i, op_len, this.data_layout().endian)) diff --git a/src/tools/miri/src/shims/unix/foreign_items.rs b/src/tools/miri/src/shims/unix/foreign_items.rs index 4931f3688572e..14297845d3dbb 100644 --- a/src/tools/miri/src/shims/unix/foreign_items.rs +++ b/src/tools/miri/src/shims/unix/foreign_items.rs @@ -593,7 +593,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { #[allow(deprecated)] let home_dir = std::env::home_dir().unwrap(); let (written, _) = this.write_path_to_c_str(&home_dir, buf, buflen)?; - let pw_dir = this.mplace_field_named(&pwd, "pw_dir")?; + let pw_dir = this.project_field_named(&pwd, "pw_dir")?; this.write_pointer(buf, &pw_dir.into())?; if written { diff --git a/src/tools/miri/src/shims/unix/fs.rs b/src/tools/miri/src/shims/unix/fs.rs index 0fdd55b407cd1..3da6c17f3b0b7 100644 --- a/src/tools/miri/src/shims/unix/fs.rs +++ b/src/tools/miri/src/shims/unix/fs.rs @@ -1141,7 +1141,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { ("tv_sec", access_sec.into()), ("tv_nsec", access_nsec.into()), ], - &this.mplace_field_named(&statxbuf, "stx_atime")?, + &this.project_field_named(&statxbuf, "stx_atime")?, )?; #[rustfmt::skip] this.write_int_fields_named( @@ -1149,7 +1149,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { ("tv_sec", created_sec.into()), ("tv_nsec", created_nsec.into()), ], - &this.mplace_field_named(&statxbuf, "stx_btime")?, + &this.project_field_named(&statxbuf, "stx_btime")?, )?; #[rustfmt::skip] this.write_int_fields_named( @@ -1157,7 +1157,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { ("tv_sec", 0.into()), ("tv_nsec", 0.into()), ], - &this.mplace_field_named(&statxbuf, "stx_ctime")?, + &this.project_field_named(&statxbuf, "stx_ctime")?, )?; #[rustfmt::skip] this.write_int_fields_named( @@ -1165,7 +1165,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { ("tv_sec", modified_sec.into()), ("tv_nsec", modified_nsec.into()), ], - &this.mplace_field_named(&statxbuf, "stx_mtime")?, + &this.project_field_named(&statxbuf, "stx_mtime")?, )?; Ok(0) @@ -1421,7 +1421,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { // } let entry_place = this.deref_operand_as(entry_op, this.libc_ty_layout("dirent"))?; - let name_place = this.mplace_field(&entry_place, 5)?; + let name_place = this.project_field(&entry_place, 5)?; let file_name = dir_entry.file_name(); // not a Path as there are no separators! let (name_fits, file_name_buf_len) = this.write_os_str_to_c_str( diff --git a/src/tools/miri/src/shims/unix/linux/fd.rs b/src/tools/miri/src/shims/unix/linux/fd.rs index 87e887000c523..9c43651132b7f 100644 --- a/src/tools/miri/src/shims/unix/linux/fd.rs +++ b/src/tools/miri/src/shims/unix/linux/fd.rs @@ -73,9 +73,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { if op == epoll_ctl_add || op == epoll_ctl_mod { let event = this.deref_operand_as(event, this.libc_ty_layout("epoll_event"))?; - let events = this.mplace_field(&event, 0)?; + let events = this.project_field(&event, 0)?; let events = this.read_scalar(&events.into())?.to_u32()?; - let data = this.mplace_field(&event, 1)?; + let data = this.project_field(&event, 1)?; let data = this.read_scalar(&data.into())?; let event = EpollEvent { events, data }; diff --git a/src/tools/miri/src/shims/unix/linux/sync.rs b/src/tools/miri/src/shims/unix/linux/sync.rs index 873b84e30599d..0474c9fd90a5c 100644 --- a/src/tools/miri/src/shims/unix/linux/sync.rs +++ b/src/tools/miri/src/shims/unix/linux/sync.rs @@ -85,7 +85,8 @@ pub fn futex<'tcx>( return Ok(()); } - let timeout = this.deref_pointer_as( + // `read_timespec` will check the place when it is not null. + let timeout = this.deref_pointer_unchecked( &this.read_immediate(&args[3])?, this.libc_ty_layout("timespec"), )?; diff --git a/src/tools/miri/src/shims/windows/foreign_items.rs b/src/tools/miri/src/shims/windows/foreign_items.rs index 2888424c21e1c..d64aa53ed9589 100644 --- a/src/tools/miri/src/shims/windows/foreign_items.rs +++ b/src/tools/miri/src/shims/windows/foreign_items.rs @@ -122,7 +122,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { // We have to put the result into io_status_block. if let Some(n) = written { let io_status_information = - this.mplace_field_named(&io_status_block, "Information")?; + this.project_field_named(&io_status_block, "Information")?; this.write_scalar( Scalar::from_target_usize(n.into(), this), &io_status_information.into(), diff --git a/src/tools/miri/tests/pass/intrinsics.rs b/src/tools/miri/tests/pass/intrinsics.rs index 6267e6e6bc111..a6a4a06f8600b 100644 --- a/src/tools/miri/tests/pass/intrinsics.rs +++ b/src/tools/miri/tests/pass/intrinsics.rs @@ -3,7 +3,7 @@ //! Tests for various intrinsics that do not fit anywhere else. use std::intrinsics; -use std::mem::{size_of, size_of_val, size_of_val_raw}; +use std::mem::{size_of, size_of_val, size_of_val_raw, discriminant}; struct Bomb; @@ -39,4 +39,7 @@ fn main() { let _v = intrinsics::discriminant_value(&0); let _v = intrinsics::discriminant_value(&true); let _v = intrinsics::discriminant_value(&vec![1, 2, 3]); + // Make sure that even if the discriminant is stored together with data, the intrinsic returns + // only the discriminant, nothing about the data. + assert_eq!(discriminant(&Some(false)), discriminant(&Some(true))); } diff --git a/tests/mir-opt/const_allocation.main.ConstProp.after.32bit.mir b/tests/mir-opt/const_allocation.main.ConstProp.after.32bit.mir index 3797dbabb3c1b..8c8e69595950d 100644 --- a/tests/mir-opt/const_allocation.main.ConstProp.after.32bit.mir +++ b/tests/mir-opt/const_allocation.main.ConstProp.after.32bit.mir @@ -23,17 +23,17 @@ alloc1 (static: FOO, size: 8, align: 4) { alloc19 (size: 48, align: 4) { 0x00 │ 00 00 00 00 __ __ __ __ ╾─alloc6──╼ 00 00 00 00 │ ....░░░░╾──╼.... - 0x10 │ 00 00 00 00 __ __ __ __ ╾─alloc9──╼ 02 00 00 00 │ ....░░░░╾──╼.... - 0x20 │ 01 00 00 00 2a 00 00 00 ╾─alloc14─╼ 03 00 00 00 │ ....*...╾──╼.... + 0x10 │ 00 00 00 00 __ __ __ __ ╾─alloc10─╼ 02 00 00 00 │ ....░░░░╾──╼.... + 0x20 │ 01 00 00 00 2a 00 00 00 ╾─alloc15─╼ 03 00 00 00 │ ....*...╾──╼.... } alloc6 (size: 0, align: 4) {} -alloc9 (size: 16, align: 4) { - ╾─alloc10─╼ 03 00 00 00 ╾─alloc11─╼ 03 00 00 00 │ ╾──╼....╾──╼.... +alloc10 (size: 16, align: 4) { + ╾─alloc9──╼ 03 00 00 00 ╾─alloc11─╼ 03 00 00 00 │ ╾──╼....╾──╼.... } -alloc10 (size: 3, align: 1) { +alloc9 (size: 3, align: 1) { 66 6f 6f │ foo } @@ -41,12 +41,12 @@ alloc11 (size: 3, align: 1) { 62 61 72 │ bar } -alloc14 (size: 24, align: 4) { - 0x00 │ ╾─alloc15─╼ 03 00 00 00 ╾─alloc16─╼ 03 00 00 00 │ ╾──╼....╾──╼.... +alloc15 (size: 24, align: 4) { + 0x00 │ ╾─alloc14─╼ 03 00 00 00 ╾─alloc16─╼ 03 00 00 00 │ ╾──╼....╾──╼.... 0x10 │ ╾─alloc17─╼ 04 00 00 00 │ ╾──╼.... } -alloc15 (size: 3, align: 1) { +alloc14 (size: 3, align: 1) { 6d 65 68 │ meh } diff --git a/tests/mir-opt/const_allocation.main.ConstProp.after.64bit.mir b/tests/mir-opt/const_allocation.main.ConstProp.after.64bit.mir index dc16c064009d7..e22547032967f 100644 --- a/tests/mir-opt/const_allocation.main.ConstProp.after.64bit.mir +++ b/tests/mir-opt/const_allocation.main.ConstProp.after.64bit.mir @@ -24,19 +24,19 @@ alloc1 (static: FOO, size: 16, align: 8) { alloc19 (size: 72, align: 8) { 0x00 │ 00 00 00 00 __ __ __ __ ╾───────alloc6────────╼ │ ....░░░░╾──────╼ 0x10 │ 00 00 00 00 00 00 00 00 00 00 00 00 __ __ __ __ │ ............░░░░ - 0x20 │ ╾───────alloc9────────╼ 02 00 00 00 00 00 00 00 │ ╾──────╼........ - 0x30 │ 01 00 00 00 2a 00 00 00 ╾───────alloc14───────╼ │ ....*...╾──────╼ + 0x20 │ ╾───────alloc10───────╼ 02 00 00 00 00 00 00 00 │ ╾──────╼........ + 0x30 │ 01 00 00 00 2a 00 00 00 ╾───────alloc15───────╼ │ ....*...╾──────╼ 0x40 │ 03 00 00 00 00 00 00 00 │ ........ } alloc6 (size: 0, align: 8) {} -alloc9 (size: 32, align: 8) { - 0x00 │ ╾───────alloc10───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........ +alloc10 (size: 32, align: 8) { + 0x00 │ ╾───────alloc9────────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........ 0x10 │ ╾───────alloc11───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........ } -alloc10 (size: 3, align: 1) { +alloc9 (size: 3, align: 1) { 66 6f 6f │ foo } @@ -44,13 +44,13 @@ alloc11 (size: 3, align: 1) { 62 61 72 │ bar } -alloc14 (size: 48, align: 8) { - 0x00 │ ╾───────alloc15───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........ +alloc15 (size: 48, align: 8) { + 0x00 │ ╾───────alloc14───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........ 0x10 │ ╾───────alloc16───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........ 0x20 │ ╾───────alloc17───────╼ 04 00 00 00 00 00 00 00 │ ╾──────╼........ } -alloc15 (size: 3, align: 1) { +alloc14 (size: 3, align: 1) { 6d 65 68 │ meh } diff --git a/tests/ui/consts/const-eval/raw-bytes.32bit.stderr b/tests/ui/consts/const-eval/raw-bytes.32bit.stderr index c0ddaceea4ce3..8a5424b3a6c67 100644 --- a/tests/ui/consts/const-eval/raw-bytes.32bit.stderr +++ b/tests/ui/consts/const-eval/raw-bytes.32bit.stderr @@ -24,7 +24,7 @@ error[E0080]: it is undefined behavior to use this value --> $DIR/raw-bytes.rs:42:1 | LL | const BAD_UNINHABITED_VARIANT1: UninhDiscriminant = unsafe { mem::transmute(1u8) }; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0: encountered a value of the never type `!` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: 1, align: 1) { @@ -35,7 +35,7 @@ error[E0080]: it is undefined behavior to use this value --> $DIR/raw-bytes.rs:44:1 | LL | const BAD_UNINHABITED_VARIANT2: UninhDiscriminant = unsafe { mem::transmute(3u8) }; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0: encountered a value of uninhabited type `Never` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: 1, align: 1) { diff --git a/tests/ui/consts/const-eval/raw-bytes.64bit.stderr b/tests/ui/consts/const-eval/raw-bytes.64bit.stderr index 20c905878e07c..08b98b37bd819 100644 --- a/tests/ui/consts/const-eval/raw-bytes.64bit.stderr +++ b/tests/ui/consts/const-eval/raw-bytes.64bit.stderr @@ -24,7 +24,7 @@ error[E0080]: it is undefined behavior to use this value --> $DIR/raw-bytes.rs:42:1 | LL | const BAD_UNINHABITED_VARIANT1: UninhDiscriminant = unsafe { mem::transmute(1u8) }; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0: encountered a value of the never type `!` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: 1, align: 1) { @@ -35,7 +35,7 @@ error[E0080]: it is undefined behavior to use this value --> $DIR/raw-bytes.rs:44:1 | LL | const BAD_UNINHABITED_VARIANT2: UninhDiscriminant = unsafe { mem::transmute(3u8) }; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0: encountered a value of uninhabited type `Never` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: 1, align: 1) { diff --git a/tests/ui/consts/const-eval/ub-enum.32bit.stderr b/tests/ui/consts/const-eval/ub-enum.32bit.stderr index 1810600b78514..5ef0d0146f2f5 100644 --- a/tests/ui/consts/const-eval/ub-enum.32bit.stderr +++ b/tests/ui/consts/const-eval/ub-enum.32bit.stderr @@ -75,7 +75,7 @@ error[E0080]: it is undefined behavior to use this value --> $DIR/ub-enum.rs:81:1 | LL | const BAD_UNINHABITED_VARIANT1: UninhDiscriminant = unsafe { mem::transmute(1u8) }; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0: encountered a value of the never type `!` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: $SIZE, align: $ALIGN) { @@ -86,7 +86,7 @@ error[E0080]: it is undefined behavior to use this value --> $DIR/ub-enum.rs:83:1 | LL | const BAD_UNINHABITED_VARIANT2: UninhDiscriminant = unsafe { mem::transmute(3u8) }; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0: encountered a value of uninhabited type `Never` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: $SIZE, align: $ALIGN) { @@ -108,14 +108,27 @@ error[E0080]: evaluation of constant value failed --> $DIR/ub-enum.rs:96:77 | LL | const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(0u64) }; - | ^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0.1: encountered a value of uninhabited type `Never` + | ^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant error[E0080]: evaluation of constant value failed --> $DIR/ub-enum.rs:98:77 | LL | const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u64) }; - | ^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0.1: encountered a value of the never type `!` + | ^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant -error: aborting due to 13 previous errors +error[E0080]: evaluation of constant value failed + --> $SRC_DIR/core/src/mem/mod.rs:LL:COL + | + = note: read discriminant of an uninhabited enum variant + | +note: inside `discriminant::` + --> $SRC_DIR/core/src/mem/mod.rs:LL:COL +note: inside `TEST_ICE_89765` + --> $DIR/ub-enum.rs:103:14 + | +LL | unsafe { std::mem::discriminant(&*(&() as *const () as *const Never)); }; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to 14 previous errors For more information about this error, try `rustc --explain E0080`. diff --git a/tests/ui/consts/const-eval/ub-enum.64bit.stderr b/tests/ui/consts/const-eval/ub-enum.64bit.stderr index fb40babb0b9c3..c28a1b722aef6 100644 --- a/tests/ui/consts/const-eval/ub-enum.64bit.stderr +++ b/tests/ui/consts/const-eval/ub-enum.64bit.stderr @@ -75,7 +75,7 @@ error[E0080]: it is undefined behavior to use this value --> $DIR/ub-enum.rs:81:1 | LL | const BAD_UNINHABITED_VARIANT1: UninhDiscriminant = unsafe { mem::transmute(1u8) }; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0: encountered a value of the never type `!` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: $SIZE, align: $ALIGN) { @@ -86,7 +86,7 @@ error[E0080]: it is undefined behavior to use this value --> $DIR/ub-enum.rs:83:1 | LL | const BAD_UNINHABITED_VARIANT2: UninhDiscriminant = unsafe { mem::transmute(3u8) }; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0: encountered a value of uninhabited type `Never` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: $SIZE, align: $ALIGN) { @@ -108,14 +108,27 @@ error[E0080]: evaluation of constant value failed --> $DIR/ub-enum.rs:96:77 | LL | const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(0u64) }; - | ^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0.1: encountered a value of uninhabited type `Never` + | ^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant error[E0080]: evaluation of constant value failed --> $DIR/ub-enum.rs:98:77 | LL | const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u64) }; - | ^^^^^^^^^^^^^^^^^^^^ constructing invalid value at ..0.1: encountered a value of the never type `!` + | ^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant -error: aborting due to 13 previous errors +error[E0080]: evaluation of constant value failed + --> $SRC_DIR/core/src/mem/mod.rs:LL:COL + | + = note: read discriminant of an uninhabited enum variant + | +note: inside `discriminant::` + --> $SRC_DIR/core/src/mem/mod.rs:LL:COL +note: inside `TEST_ICE_89765` + --> $DIR/ub-enum.rs:103:14 + | +LL | unsafe { std::mem::discriminant(&*(&() as *const () as *const Never)); }; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to 14 previous errors For more information about this error, try `rustc --explain E0080`. diff --git a/tests/ui/consts/const-eval/ub-enum.rs b/tests/ui/consts/const-eval/ub-enum.rs index 8f26d9a00d71b..078283fbd1f7d 100644 --- a/tests/ui/consts/const-eval/ub-enum.rs +++ b/tests/ui/consts/const-eval/ub-enum.rs @@ -2,7 +2,7 @@ // Strip out raw byte dumps to make comparison platform-independent: // normalize-stderr-test "(the raw bytes of the constant) \(size: [0-9]*, align: [0-9]*\)" -> "$1 (size: $$SIZE, align: $$ALIGN)" // normalize-stderr-test "([0-9a-f][0-9a-f] |╾─*a(lloc)?[0-9]+(\+[a-z0-9]+)?─*╼ )+ *│.*" -> "HEX_DUMP" -#![feature(never_type)] +#![feature(never_type, const_discriminant)] #![allow(invalid_value)] use std::mem; @@ -66,8 +66,8 @@ const BAD_ENUM2_OPTION_PTR: Option = unsafe { mem::transmute(&0) }; // # valid discriminant for uninhabited variant -// An enum with 3 variants of which some are uninhabited -- so the uninhabited variants *do* -// have a discriminant. +// An enum with uninhabited variants but also at least 2 inhabited variants -- so the uninhabited +// variants *do* have a discriminant. enum UninhDiscriminant { A, B(!), @@ -98,5 +98,11 @@ const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem: const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u64) }; //~^ ERROR evaluation of constant value failed +const TEST_ICE_89765: () = { + // This is a regression test for https://github.com/rust-lang/rust/issues/89765. + unsafe { std::mem::discriminant(&*(&() as *const () as *const Never)); }; + //~^ inside `TEST_ICE_89765` +}; + fn main() { } diff --git a/tests/ui/consts/const-eval/ub-ref-ptr.stderr b/tests/ui/consts/const-eval/ub-ref-ptr.stderr index 1d19dfff50b36..d1644f8a4dc1f 100644 --- a/tests/ui/consts/const-eval/ub-ref-ptr.stderr +++ b/tests/ui/consts/const-eval/ub-ref-ptr.stderr @@ -141,7 +141,7 @@ error[E0080]: it is undefined behavior to use this value --> $DIR/ub-ref-ptr.rs:59:1 | LL | const DATA_FN_PTR: fn() = unsafe { mem::transmute(&13) }; - | ^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value: encountered alloc41, but expected a function pointer + | ^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value: encountered alloc39, but expected a function pointer | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: $SIZE, align: $ALIGN) { diff --git a/tests/ui/consts/const_discriminant.rs b/tests/ui/consts/const_discriminant.rs index 79e68590e85d4..b1180faa69776 100644 --- a/tests/ui/consts/const_discriminant.rs +++ b/tests/ui/consts/const_discriminant.rs @@ -24,13 +24,6 @@ enum SingleVariant { const TEST_V: Discriminant = discriminant(&SingleVariant::V); -pub const TEST_VOID: () = { - // This is UB, but CTFE does not check validity so it does not detect this. - // This is a regression test for https://github.com/rust-lang/rust/issues/89765. - unsafe { std::mem::discriminant(&*(&() as *const () as *const Void)); }; -}; - - fn main() { assert_eq!(TEST_A, TEST_A_OTHER); assert_eq!(TEST_A, discriminant(black_box(&Test::A(17)))); diff --git a/tests/ui/consts/extra-const-ub/detect-extra-ub.rs b/tests/ui/consts/extra-const-ub/detect-extra-ub.rs index 6a3c93ce7a674..5bff34bbe9338 100644 --- a/tests/ui/consts/extra-const-ub/detect-extra-ub.rs +++ b/tests/ui/consts/extra-const-ub/detect-extra-ub.rs @@ -1,8 +1,26 @@ // revisions: no_flag with_flag // [no_flag] check-pass // [with_flag] compile-flags: -Zextra-const-ub-checks +#![feature(never_type)] use std::mem::transmute; +use std::ptr::addr_of; + +#[derive(Clone, Copy)] +enum E { A, B } + +#[derive(Clone, Copy)] +enum Never {} + +// An enum with uninhabited variants but also at least 2 inhabited variants -- so the uninhabited +// variants *do* have a discriminant. +#[derive(Clone, Copy)] +enum UninhDiscriminant { + A, + B(!), + C, + D(Never), +} const INVALID_BOOL: () = unsafe { let _x: bool = transmute(3u8); @@ -27,4 +45,15 @@ const UNALIGNED_PTR: () = unsafe { //[with_flag]~| invalid value }; +const UNINHABITED_VARIANT: () = unsafe { + let data = [1u8]; + // Not using transmute, we want to hit the ImmTy code path. + let v = *addr_of!(data).cast::(); + //[with_flag]~^ ERROR: evaluation of constant value failed +}; + +// Regression tests for an ICE (related to ). +const VALID_ENUM1: E = { let e = E::A; e }; +const VALID_ENUM2: Result<&'static [u8], ()> = { let e = Err(()); e }; + fn main() {} diff --git a/tests/ui/consts/extra-const-ub/detect-extra-ub.with_flag.stderr b/tests/ui/consts/extra-const-ub/detect-extra-ub.with_flag.stderr index 3970baefcb354..19f1748ff9c96 100644 --- a/tests/ui/consts/extra-const-ub/detect-extra-ub.with_flag.stderr +++ b/tests/ui/consts/extra-const-ub/detect-extra-ub.with_flag.stderr @@ -1,11 +1,11 @@ error[E0080]: evaluation of constant value failed - --> $DIR/detect-extra-ub.rs:8:20 + --> $DIR/detect-extra-ub.rs:26:20 | LL | let _x: bool = transmute(3u8); | ^^^^^^^^^^^^^^ constructing invalid value: encountered 0x03, but expected a boolean error[E0080]: evaluation of constant value failed - --> $DIR/detect-extra-ub.rs:14:21 + --> $DIR/detect-extra-ub.rs:32:21 | LL | let _x: usize = transmute(&3u8); | ^^^^^^^^^^^^^^^ unable to turn pointer into raw bytes @@ -14,7 +14,7 @@ LL | let _x: usize = transmute(&3u8); = help: the absolute address of a pointer is not known at compile-time, so such operations are not supported error[E0080]: evaluation of constant value failed - --> $DIR/detect-extra-ub.rs:20:30 + --> $DIR/detect-extra-ub.rs:38:30 | LL | let _x: (usize, usize) = transmute(x); | ^^^^^^^^^^^^ unable to turn pointer into raw bytes @@ -23,11 +23,17 @@ LL | let _x: (usize, usize) = transmute(x); = help: the absolute address of a pointer is not known at compile-time, so such operations are not supported error[E0080]: evaluation of constant value failed - --> $DIR/detect-extra-ub.rs:25:20 + --> $DIR/detect-extra-ub.rs:43:20 | LL | let _x: &u32 = transmute(&[0u8; 4]); | ^^^^^^^^^^^^^^^^^^^^ constructing invalid value: encountered an unaligned reference (required 4 byte alignment but found 1) -error: aborting due to 4 previous errors +error[E0080]: evaluation of constant value failed + --> $DIR/detect-extra-ub.rs:51:13 + | +LL | let v = *addr_of!(data).cast::(); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ constructing invalid value at .: encountered an uninhabited enum variant + +error: aborting due to 5 previous errors For more information about this error, try `rustc --explain E0080`.