Skip to content

Commit

Permalink
Auto merge of #95224 - mjbshaw:patch-1, r=yaahc
Browse files Browse the repository at this point in the history
Optimize RcInnerPtr::inc_strong()/inc_weak() instruction count

Inspired by this internals thread: https://internals.rust-lang.org/t/rc-optimization-on-64-bit-targets/16362

[The generated assembly is a bit smaller](https://rust.godbolt.org/z/TeTnf6144) and is a more efficient usage of the CPU's instruction cache. `unlikely` doesn't impact any of the small artificial tests I've done, but I've included it in case it might help more complex scenarios when this is inlined.
  • Loading branch information
bors committed Apr 15, 2022
2 parents 4886d8b + 7fad6f1 commit bea2b2b
Showing 1 changed file with 28 additions and 10 deletions.
38 changes: 28 additions & 10 deletions alloc/src/rc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2511,14 +2511,23 @@ trait RcInnerPtr {
fn inc_strong(&self) {
let strong = self.strong();

// We insert an `assume` here to hint LLVM at an otherwise
// missed optimization.
// SAFETY: The reference count will never be zero when this is
// called.
unsafe {
core::intrinsics::assume(strong != 0);
}

let strong = strong.wrapping_add(1);
self.strong_ref().set(strong);

// We want to abort on overflow instead of dropping the value.
// The reference count will never be zero when this is called;
// nevertheless, we insert an abort here to hint LLVM at
// an otherwise missed optimization.
if strong == 0 || strong == usize::MAX {
// Checking for overflow after the store instead of before
// allows for slightly better code generation.
if core::intrinsics::unlikely(strong == 0) {
abort();
}
self.strong_ref().set(strong + 1);
}

#[inline]
Expand All @@ -2535,14 +2544,23 @@ trait RcInnerPtr {
fn inc_weak(&self) {
let weak = self.weak();

// We insert an `assume` here to hint LLVM at an otherwise
// missed optimization.
// SAFETY: The reference count will never be zero when this is
// called.
unsafe {
core::intrinsics::assume(weak != 0);
}

let weak = weak.wrapping_add(1);
self.weak_ref().set(weak);

// We want to abort on overflow instead of dropping the value.
// The reference count will never be zero when this is called;
// nevertheless, we insert an abort here to hint LLVM at
// an otherwise missed optimization.
if weak == 0 || weak == usize::MAX {
// Checking for overflow after the store instead of before
// allows for slightly better code generation.
if core::intrinsics::unlikely(weak == 0) {
abort();
}
self.weak_ref().set(weak + 1);
}

#[inline]
Expand Down

0 comments on commit bea2b2b

Please sign in to comment.