Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pass large interpreter types by reference, not value #82124

Merged
merged 6 commits into from
Feb 20, 2021
Merged

Pass large interpreter types by reference, not value #82124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6c9d7fb
Add size assertions for interpreter data structures
tmiasko Feb 14, 2021
e915cf4
Pass OpTy by reference not value
tmiasko Feb 15, 2021
fe0c46d
Pass PlaceTy by reference not value
tmiasko Feb 15, 2021
5888556
Pass ImmTy by reference not value
tmiasko Feb 15, 2021
d06a2a3
Pass MPlaceTy by reference not value
tmiasko Feb 15, 2021
f2da425
./x.py fmt
tmiasko Feb 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions compiler/rustc_mir/src/const_eval/eval_queries.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
ecx.push_stack_frame(
cid.instance,
body,
Some(ret.into()),
Some(&ret.into()),
StackPopCleanup::None { cleanup: false },
)?;

Expand All @@ -72,7 +72,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
None => InternKind::Constant,
}
};
intern_const_alloc_recursive(ecx, intern_kind, ret)?;
intern_const_alloc_recursive(ecx, intern_kind, &ret)?;

debug!("eval_body_using_ecx done: {:?}", *ret);
Ok(ret)
Expand Down Expand Up @@ -105,7 +105,7 @@ pub(super) fn mk_eval_cx<'mir, 'tcx>(
/// type system.
pub(super) fn op_to_const<'tcx>(
ecx: &CompileTimeEvalContext<'_, 'tcx>,
op: OpTy<'tcx>,
op: &OpTy<'tcx>,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This kind of changes should already happen at ABI level.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Couldn't it still make a difference on the caller side, which might have to worry about the callee mutating that memory when things are implicitly pass-by-ref?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I think that is the case.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The OpTy is classified as a memory (see commit with size assertions) and is passed on the stack. If a function is merely forwarding an OpTy argument to another, it will have to copy it into a stack slot. What could be improved there?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, you are right about rust ABI. We are passing those indirectly without byval, but that means memcpyopt cannot eliminate the copy.

) -> ConstValue<'tcx> {
// We do not have value optimizations for everything.
// Only scalars and slices, since they are very common.
Expand Down Expand Up @@ -137,7 +137,7 @@ pub(super) fn op_to_const<'tcx>(
op.try_as_mplace(ecx)
};

let to_const_value = |mplace: MPlaceTy<'_>| match mplace.ptr {
let to_const_value = |mplace: &MPlaceTy<'_>| match mplace.ptr {
Scalar::Ptr(ptr) => {
let alloc = ecx.tcx.global_alloc(ptr.alloc_id).unwrap_memory();
ConstValue::ByRef { alloc, offset: ptr.offset }
Expand All @@ -155,12 +155,12 @@ pub(super) fn op_to_const<'tcx>(
}
};
match immediate {
Ok(mplace) => to_const_value(mplace),
Ok(ref mplace) => to_const_value(mplace),
// see comment on `let try_as_immediate` above
Err(imm) => match *imm {
Immediate::Scalar(x) => match x {
ScalarMaybeUninit::Scalar(s) => ConstValue::Scalar(s),
ScalarMaybeUninit::Uninit => to_const_value(op.assert_mem_place(ecx)),
ScalarMaybeUninit::Uninit => to_const_value(&op.assert_mem_place(ecx)),
},
Immediate::ScalarPair(a, b) => {
let (data, start) = match a.check_init().unwrap() {
Expand Down Expand Up @@ -201,7 +201,7 @@ fn turn_into_const_value<'tcx>(
"the `eval_to_const_value_raw` query should not be used for statics, use `eval_to_allocation` instead"
);
// Turn this into a proper constant.
op_to_const(&ecx, mplace.into())
op_to_const(&ecx, &mplace.into())
}

pub fn eval_to_const_value_raw_provider<'tcx>(
Expand Down Expand Up @@ -348,7 +348,7 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
Some(_) => CtfeValidationMode::Regular, // a `static`
None => CtfeValidationMode::Const { inner, allow_static_ptrs: false },
};
ecx.const_validate_operand(mplace.into(), path, &mut ref_tracking, mode)?;
ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)?;
inner = true;
}
};
Expand Down
24 changes: 12 additions & 12 deletions compiler/rustc_mir/src/const_eval/machine.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> {
// &str
assert!(args.len() == 1);

let msg_place = self.deref_operand(args[0])?;
let msg = Symbol::intern(self.read_str(msg_place)?);
let msg_place = self.deref_operand(&args[0])?;
let msg = Symbol::intern(self.read_str(&msg_place)?);
let span = self.find_closest_untracked_caller_location();
let (file, line, col) = self.location_triple_for_span(span);
Err(ConstEvalErrKind::Panic { msg, file, line, col }.into())
Expand Down Expand Up @@ -222,7 +222,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
instance: ty::Instance<'tcx>,
_abi: Abi,
args: &[OpTy<'tcx>],
_ret: Option<(PlaceTy<'tcx>, mir::BasicBlock)>,
_ret: Option<(&PlaceTy<'tcx>, mir::BasicBlock)>,
_unwind: Option<mir::BasicBlock>, // unwinding is not supported in consts
) -> InterpResult<'tcx, Option<&'mir mir::Body<'tcx>>> {
debug!("find_mir_or_eval_fn: {:?}", instance);
Expand Down Expand Up @@ -262,7 +262,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
ecx: &mut InterpCx<'mir, 'tcx, Self>,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx>],
ret: Option<(PlaceTy<'tcx>, mir::BasicBlock)>,
ret: Option<(&PlaceTy<'tcx>, mir::BasicBlock)>,
_unwind: Option<mir::BasicBlock>,
) -> InterpResult<'tcx> {
// Shared intrinsics.
Expand All @@ -284,8 +284,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
};
match intrinsic_name {
sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => {
let a = ecx.read_immediate(args[0])?.to_scalar()?;
let b = ecx.read_immediate(args[1])?.to_scalar()?;
let a = ecx.read_immediate(&args[0])?.to_scalar()?;
let b = ecx.read_immediate(&args[1])?.to_scalar()?;
let cmp = if intrinsic_name == sym::ptr_guaranteed_eq {
ecx.guaranteed_eq(a, b)
} else {
Expand All @@ -294,8 +294,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
ecx.write_scalar(Scalar::from_bool(cmp), dest)?;
}
sym::const_allocate => {
let size = ecx.read_scalar(args[0])?.to_machine_usize(ecx)?;
let align = ecx.read_scalar(args[1])?.to_machine_usize(ecx)?;
let size = ecx.read_scalar(&args[0])?.to_machine_usize(ecx)?;
let align = ecx.read_scalar(&args[1])?.to_machine_usize(ecx)?;

let align = match Align::from_bytes(align) {
Ok(a) => a,
Expand Down Expand Up @@ -330,7 +330,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
use rustc_middle::mir::AssertKind::*;
// Convert `AssertKind<Operand>` to `AssertKind<Scalar>`.
let eval_to_int =
|op| ecx.read_immediate(ecx.eval_operand(op, None)?).map(|x| x.to_const_int());
|op| ecx.read_immediate(&ecx.eval_operand(op, None)?).map(|x| x.to_const_int());
let err = match msg {
BoundsCheck { ref len, ref index } => {
let len = eval_to_int(len)?;
Expand Down Expand Up @@ -358,15 +358,15 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
fn binary_ptr_op(
_ecx: &InterpCx<'mir, 'tcx, Self>,
_bin_op: mir::BinOp,
_left: ImmTy<'tcx>,
_right: ImmTy<'tcx>,
_left: &ImmTy<'tcx>,
_right: &ImmTy<'tcx>,
) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> {
Err(ConstEvalErrKind::NeedsRfc("pointer arithmetic or comparison".to_string()).into())
}

fn box_alloc(
_ecx: &mut InterpCx<'mir, 'tcx, Self>,
_dest: PlaceTy<'tcx>,
_dest: &PlaceTy<'tcx>,
) -> InterpResult<'tcx> {
Err(ConstEvalErrKind::NeedsRfc("heap allocations via `box` keyword".to_string()).into())
}
Expand Down
14 changes: 7 additions & 7 deletions compiler/rustc_mir/src/const_eval/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ pub(crate) fn const_caller_location(
let mut ecx = mk_eval_cx(tcx, DUMMY_SP, ty::ParamEnv::reveal_all(), false);

let loc_place = ecx.alloc_caller_location(file, line, col);
if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, loc_place).is_err() {
if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &loc_place).is_err() {
bug!("intern_const_alloc_recursive should not error in this case")
}
ConstValue::Scalar(loc_place.ptr)
Expand All @@ -55,17 +55,17 @@ pub(crate) fn destructure_const<'tcx>(
return mir::DestructuredConst { variant: None, fields: &[] };
}
ty::Adt(def, _) => {
let variant = ecx.read_discriminant(op).unwrap().1;
let down = ecx.operand_downcast(op, variant).unwrap();
let variant = ecx.read_discriminant(&op).unwrap().1;
let down = ecx.operand_downcast(&op, variant).unwrap();
(def.variants[variant].fields.len(), Some(variant), down)
}
ty::Tuple(substs) => (substs.len(), None, op),
_ => bug!("cannot destructure constant {:?}", val),
};

let fields_iter = (0..field_count).map(|i| {
let field_op = ecx.operand_field(down, i).unwrap();
let val = op_to_const(&ecx, field_op);
let field_op = ecx.operand_field(&down, i).unwrap();
let val = op_to_const(&ecx, &field_op);
ty::Const::from_value(tcx, val, field_op.layout.ty)
});
let fields = tcx.arena.alloc_from_iter(fields_iter);
Expand All @@ -81,7 +81,7 @@ pub(crate) fn deref_const<'tcx>(
trace!("deref_const: {:?}", val);
let ecx = mk_eval_cx(tcx, DUMMY_SP, param_env, false);
let op = ecx.const_to_op(val, None).unwrap();
let mplace = ecx.deref_operand(op).unwrap();
let mplace = ecx.deref_operand(&op).unwrap();
if let Scalar::Ptr(ptr) = mplace.ptr {
assert_eq!(
ecx.memory.get_raw(ptr.alloc_id).unwrap().mutability,
Expand All @@ -106,5 +106,5 @@ pub(crate) fn deref_const<'tcx>(
},
};

tcx.mk_const(ty::Const { val: ty::ConstKind::Value(op_to_const(&ecx, mplace.into())), ty })
tcx.mk_const(ty::Const { val: ty::ConstKind::Value(op_to_const(&ecx, &mplace.into())), ty })
}
24 changes: 12 additions & 12 deletions compiler/rustc_mir/src/interpret/cast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ use super::{
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub fn cast(
&mut self,
src: OpTy<'tcx, M::PointerTag>,
src: &OpTy<'tcx, M::PointerTag>,
cast_kind: CastKind,
cast_ty: Ty<'tcx>,
dest: PlaceTy<'tcx, M::PointerTag>,
dest: &PlaceTy<'tcx, M::PointerTag>,
) -> InterpResult<'tcx> {
use rustc_middle::mir::CastKind::*;
// FIXME: In which cases should we trigger UB when the source is uninit?
Expand All @@ -32,7 +32,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {

Misc => {
let src = self.read_immediate(src)?;
let res = self.misc_cast(src, cast_ty)?;
let res = self.misc_cast(&src, cast_ty)?;
self.write_immediate(res, dest)?;
}

Expand Down Expand Up @@ -107,7 +107,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {

fn misc_cast(
&self,
src: ImmTy<'tcx, M::PointerTag>,
src: &ImmTy<'tcx, M::PointerTag>,
cast_ty: Ty<'tcx>,
) -> InterpResult<'tcx, Immediate<M::PointerTag>> {
use rustc_middle::ty::TyKind::*;
Expand Down Expand Up @@ -158,13 +158,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let dest_layout = self.layout_of(cast_ty)?;
if dest_layout.size == src.layout.size {
// Thin or fat pointer that just hast the ptr kind of target type changed.
return Ok(*src);
return Ok(**src);
} else {
// Casting the metadata away from a fat ptr.
assert_eq!(src.layout.size, 2 * self.memory.pointer_size());
assert_eq!(dest_layout.size, self.memory.pointer_size());
assert!(src.layout.ty.is_unsafe_ptr());
return match *src {
return match **src {
Immediate::ScalarPair(data, _) => Ok(data.into()),
Immediate::Scalar(..) => span_bug!(
self.cur_span(),
Expand Down Expand Up @@ -259,8 +259,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {

fn unsize_into_ptr(
&mut self,
src: OpTy<'tcx, M::PointerTag>,
dest: PlaceTy<'tcx, M::PointerTag>,
src: &OpTy<'tcx, M::PointerTag>,
dest: &PlaceTy<'tcx, M::PointerTag>,
// The pointee types
source_ty: Ty<'tcx>,
cast_ty: Ty<'tcx>,
Expand Down Expand Up @@ -300,9 +300,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {

fn unsize_into(
&mut self,
src: OpTy<'tcx, M::PointerTag>,
src: &OpTy<'tcx, M::PointerTag>,
cast_ty: TyAndLayout<'tcx>,
dest: PlaceTy<'tcx, M::PointerTag>,
dest: &PlaceTy<'tcx, M::PointerTag>,
) -> InterpResult<'tcx> {
trace!("Unsizing {:?} of type {} into {:?}", *src, src.layout.ty, cast_ty.ty);
match (&src.layout.ty.kind(), &cast_ty.ty.kind()) {
Expand Down Expand Up @@ -340,9 +340,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let src_field = self.operand_field(src, i)?;
let dst_field = self.place_field(dest, i)?;
if src_field.layout.ty == cast_ty_field.ty {
self.copy_op(src_field, dst_field)?;
self.copy_op(&src_field, &dst_field)?;
} else {
self.unsize_into(src_field, cast_ty_field, dst_field)?;
self.unsize_into(&src_field, cast_ty_field, &dst_field)?;
}
}
Ok(())
Expand Down
53 changes: 27 additions & 26 deletions compiler/rustc_mir/src/interpret/eval_context.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -548,8 +548,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// This can fail to provide an answer for extern types.
pub(super) fn size_and_align_of(
&self,
metadata: MemPlaceMeta<M::PointerTag>,
layout: TyAndLayout<'tcx>,
metadata: &MemPlaceMeta<M::PointerTag>,
layout: &TyAndLayout<'tcx>,
) -> InterpResult<'tcx, Option<(Size, Align)>> {
if !layout.is_unsized() {
return Ok(Some((layout.size, layout.align.abi)));
Expand Down Expand Up @@ -577,24 +577,25 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// the last field). Can't have foreign types here, how would we
// adjust alignment and size for them?
let field = layout.field(self, layout.fields.count() - 1)?;
let (unsized_size, unsized_align) = match self.size_and_align_of(metadata, field)? {
Some(size_and_align) => size_and_align,
None => {
// A field with extern type. If this field is at offset 0, we behave
// like the underlying extern type.
// FIXME: Once we have made decisions for how to handle size and alignment
// of `extern type`, this should be adapted. It is just a temporary hack
// to get some code to work that probably ought to work.
if sized_size == Size::ZERO {
return Ok(None);
} else {
span_bug!(
self.cur_span(),
"Fields cannot be extern types, unless they are at offset 0"
)
let (unsized_size, unsized_align) =
match self.size_and_align_of(metadata, &field)? {
Some(size_and_align) => size_and_align,
None => {
// A field with extern type. If this field is at offset 0, we behave
// like the underlying extern type.
// FIXME: Once we have made decisions for how to handle size and alignment
// of `extern type`, this should be adapted. It is just a temporary hack
// to get some code to work that probably ought to work.
if sized_size == Size::ZERO {
return Ok(None);
} else {
span_bug!(
self.cur_span(),
"Fields cannot be extern types, unless they are at offset 0"
)
}
}
}
};
};

// FIXME (#26403, #27023): We should be adding padding
// to `sized_size` (to accommodate the `unsized_align`
Expand Down Expand Up @@ -645,24 +646,24 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
#[inline]
pub fn size_and_align_of_mplace(
&self,
mplace: MPlaceTy<'tcx, M::PointerTag>,
mplace: &MPlaceTy<'tcx, M::PointerTag>,
) -> InterpResult<'tcx, Option<(Size, Align)>> {
self.size_and_align_of(mplace.meta, mplace.layout)
self.size_and_align_of(&mplace.meta, &mplace.layout)
}

pub fn push_stack_frame(
&mut self,
instance: ty::Instance<'tcx>,
body: &'mir mir::Body<'tcx>,
return_place: Option<PlaceTy<'tcx, M::PointerTag>>,
return_place: Option<&PlaceTy<'tcx, M::PointerTag>>,
return_to_block: StackPopCleanup,
) -> InterpResult<'tcx> {
// first push a stack frame so we have access to the local substs
let pre_frame = Frame {
body,
loc: Err(body.span), // Span used for errors caused during preamble.
return_to_block,
return_place,
return_place: return_place.copied(),
// empty local array, we fill it in below, after we are inside the stack frame and
// all methods actually know about the frame
locals: IndexVec::new(),
Expand Down Expand Up @@ -777,10 +778,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {

if !unwinding {
// Copy the return value to the caller's stack frame.
if let Some(return_place) = frame.return_place {
if let Some(ref return_place) = frame.return_place {
let op = self.access_local(&frame, mir::RETURN_PLACE, None)?;
self.copy_op_transmute(op, return_place)?;
trace!("{:?}", self.dump_place(*return_place));
self.copy_op_transmute(&op, return_place)?;
trace!("{:?}", self.dump_place(**return_place));
} else {
throw_ub!(Unreachable);
}
Expand Down
Loading