Skip to content

Commit

Permalink
Auto merge of #67658 - spastorino:do-not-copy-zsts, r=oli-obk
Browse files Browse the repository at this point in the history
Avoid memory copy logic for zsts

r? @oli-obk

One of the included commits is work done by @HeroicKatora in #62655
  • Loading branch information
bors committed Dec 30, 2019
2 parents d297b19 + 250a636 commit 580ac0b
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 25 deletions.
8 changes: 8 additions & 0 deletions src/librustc/mir/interpret/allocation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,14 @@ pub struct AllocationDefinedness {
ranges: smallvec::SmallVec<[u64; 1]>,
}

impl AllocationDefinedness {
pub fn all_bytes_undef(&self) -> bool {
// The `ranges` are run-length encoded and of alternating definedness.
// So if `ranges.len() > 1` then the second block is a range of defined.
self.initial == false && self.ranges.len() == 1
}
}

/// Transferring the definedness mask to other allocations.
impl<Tag, Extra> Allocation<Tag, Extra> {
/// Creates a run-length encoding of the undef mask.
Expand Down
62 changes: 37 additions & 25 deletions src/librustc_mir/interpret/memory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -841,11 +841,38 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {

let tcx = self.tcx.tcx;

// The bits have to be saved locally before writing to dest in case src and dest overlap.
assert_eq!(size.bytes() as usize as u64, size.bytes());

// This checks relocation edges on the src.
let src_bytes =
self.get_raw(src.alloc_id)?.get_bytes_with_undef_and_ptr(&tcx, src, size)?.as_ptr();
let dest_bytes =
self.get_raw_mut(dest.alloc_id)?.get_bytes_mut(&tcx, dest, size * length)?.as_mut_ptr();
self.get_raw_mut(dest.alloc_id)?.get_bytes_mut(&tcx, dest, size * length)?;

// If `dest_bytes` is empty we just optimize to not run anything for zsts.
// See #67539
if dest_bytes.is_empty() {
return Ok(());
}

let dest_bytes = dest_bytes.as_mut_ptr();

// Prepare a copy of the undef mask.
let compressed = self.get_raw(src.alloc_id)?.compress_undef_range(src, size);

if compressed.all_bytes_undef() {
// Fast path: If all bytes are `undef` then there is nothing to copy. The target range
// is marked as undef but we otherwise omit changing the byte representation which may
// be arbitrary for undef bytes.
// This also avoids writing to the target bytes so that the backing allocation is never
// touched if the bytes stay undef for the whole interpreter execution. On contemporary
// operating system this can avoid physically allocating the page.
let dest_alloc = self.get_raw_mut(dest.alloc_id)?;
dest_alloc.mark_definedness(dest, size * length, false);
dest_alloc.mark_relocation_range(relocations);
return Ok(());
}

// SAFE: The above indexing would have panicked if there weren't at least `size` bytes
// behind `src` and `dest`. Also, we use the overlapping-safe `ptr::copy` if `src` and
Expand Down Expand Up @@ -881,38 +908,23 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
}
}

// copy definedness to the destination
self.copy_undef_mask(src, dest, size, length)?;
// now fill in all the data
self.get_raw_mut(dest.alloc_id)?.mark_compressed_undef_range(
&compressed,
dest,
size,
length,
);

// copy the relocations to the destination
self.get_raw_mut(dest.alloc_id)?.mark_relocation_range(relocations);

Ok(())
}
}

/// Undefined bytes
/// Machine pointer introspection.
impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
// FIXME: Add a fast version for the common, nonoverlapping case
fn copy_undef_mask(
&mut self,
src: Pointer<M::PointerTag>,
dest: Pointer<M::PointerTag>,
size: Size,
repeat: u64,
) -> InterpResult<'tcx> {
// The bits have to be saved locally before writing to dest in case src and dest overlap.
assert_eq!(size.bytes() as usize as u64, size.bytes());

let src_alloc = self.get_raw(src.alloc_id)?;
let compressed = src_alloc.compress_undef_range(src, size);

// now fill in all the data
let dest_allocation = self.get_raw_mut(dest.alloc_id)?;
dest_allocation.mark_compressed_undef_range(&compressed, dest, size, repeat);

Ok(())
}

pub fn force_ptr(
&self,
scalar: Scalar<M::PointerTag>,
Expand Down

0 comments on commit 580ac0b

Please sign in to comment.