Minor perf (#607)

* Prefetch on traversing remote queue * Make pointer function fast path * Only try to form blocks of MIN_BITS
microsoft · Mar 24, 2023 · d8f174c · d8f174c
1 parent 8e3f6c9
commit d8f174c
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 27 deletions.
diff --git a/src/snmalloc/aal/address.h b/src/snmalloc/aal/address.h
@@ -14,7 +14,8 @@ namespace snmalloc
   /**
    * Perform arithmetic on a uintptr_t.
    */
-  inline uintptr_t pointer_offset(uintptr_t base, size_t diff)
+  SNMALLOC_FAST_PATH_INLINE uintptr_t
+  pointer_offset(uintptr_t base, size_t diff)
   {
     return base + diff;
   }
@@ -23,15 +24,15 @@ namespace snmalloc
    * Perform pointer arithmetic and return the adjusted pointer.
    */
   template<typename U = void, typename T>
-  inline U* pointer_offset(T* base, size_t diff)
+  SNMALLOC_FAST_PATH_INLINE U* pointer_offset(T* base, size_t diff)
   {
     SNMALLOC_ASSERT(base != nullptr); /* Avoid UB */
     return unsafe_from_uintptr<U>(
       unsafe_to_uintptr<T>(base) + static_cast<uintptr_t>(diff));
   }
 
   template<SNMALLOC_CONCEPT(capptr::IsBound) bounds, typename T>
-  inline CapPtr<void, bounds>
+  SNMALLOC_FAST_PATH_INLINE CapPtr<void, bounds>
   pointer_offset(CapPtr<T, bounds> base, size_t diff)
   {
     return CapPtr<void, bounds>::unsafe_from(
@@ -42,14 +43,14 @@ namespace snmalloc
    * Perform pointer arithmetic and return the adjusted pointer.
    */
   template<typename U = void, typename T>
-  inline U* pointer_offset_signed(T* base, ptrdiff_t diff)
+  SNMALLOC_FAST_PATH_INLINE U* pointer_offset_signed(T* base, ptrdiff_t diff)
   {
     SNMALLOC_ASSERT(base != nullptr); /* Avoid UB */
     return reinterpret_cast<U*>(reinterpret_cast<char*>(base) + diff);
   }
 
   template<SNMALLOC_CONCEPT(capptr::IsBound) bounds, typename T>
-  inline CapPtr<void, bounds>
+  SNMALLOC_FAST_PATH_INLINE CapPtr<void, bounds>
   pointer_offset_signed(CapPtr<T, bounds> base, ptrdiff_t diff)
   {
     return CapPtr<void, bounds>::unsafe_from(
@@ -60,7 +61,7 @@ namespace snmalloc
    * Cast from a pointer type to an address.
    */
   template<typename T>
-  inline SNMALLOC_FAST_PATH address_t address_cast(T* ptr)
+  SNMALLOC_FAST_PATH_INLINE address_t address_cast(T* ptr)
   {
     return reinterpret_cast<address_t>(ptr);
   }
@@ -73,12 +74,12 @@ namespace snmalloc
    * capptr_bound.
    */
   template<typename T, SNMALLOC_CONCEPT(capptr::IsBound) bounds>
-  inline SNMALLOC_FAST_PATH address_t address_cast(CapPtr<T, bounds> a)
+  SNMALLOC_FAST_PATH_INLINE address_t address_cast(CapPtr<T, bounds> a)
   {
     return address_cast(a.unsafe_ptr());
   }
 
-  inline SNMALLOC_FAST_PATH address_t address_cast(uintptr_t a)
+  SNMALLOC_FAST_PATH_INLINE address_t address_cast(uintptr_t a)
   {
     return static_cast<address_t>(a);
   }
@@ -88,15 +89,15 @@ namespace snmalloc
    * two.
    */
   template<size_t alignment>
-  static inline bool is_aligned_block(address_t p, size_t size)
+  SNMALLOC_FAST_PATH_INLINE bool is_aligned_block(address_t p, size_t size)
   {
     static_assert(bits::is_pow2(alignment));
 
     return ((p | size) & (alignment - 1)) == 0;
   }
 
   template<size_t alignment>
-  static inline bool is_aligned_block(void* p, size_t size)
+  SNMALLOC_FAST_PATH_INLINE bool is_aligned_block(void* p, size_t size)
   {
     return is_aligned_block<alignment>(address_cast(p), size);
   }
@@ -106,7 +107,7 @@ namespace snmalloc
    * a power of two.
    */
   template<size_t alignment>
-  inline uintptr_t pointer_align_down(uintptr_t p)
+  SNMALLOC_FAST_PATH_INLINE uintptr_t pointer_align_down(uintptr_t p)
   {
     static_assert(alignment > 0);
     static_assert(bits::is_pow2(alignment));
@@ -127,7 +128,7 @@ namespace snmalloc
    * power of two.
    */
   template<size_t alignment, typename T = void>
-  inline T* pointer_align_down(void* p)
+  SNMALLOC_FAST_PATH_INLINE T* pointer_align_down(void* p)
   {
     return unsafe_from_uintptr<T>(
       pointer_align_down<alignment>(unsafe_to_uintptr<void>(p)));
@@ -137,14 +138,15 @@ namespace snmalloc
     size_t alignment,
     typename T,
     SNMALLOC_CONCEPT(capptr::IsBound) bounds>
-  inline CapPtr<T, bounds> pointer_align_down(CapPtr<void, bounds> p)
+  SNMALLOC_FAST_PATH_INLINE CapPtr<T, bounds>
+  pointer_align_down(CapPtr<void, bounds> p)
   {
     return CapPtr<T, bounds>::unsafe_from(
       pointer_align_down<alignment, T>(p.unsafe_ptr()));
   }
 
   template<size_t alignment>
-  inline address_t address_align_down(address_t p)
+  SNMALLOC_FAST_PATH_INLINE address_t address_align_down(address_t p)
   {
     return bits::align_down(p, alignment);
   }
@@ -154,7 +156,7 @@ namespace snmalloc
    * power of two.
    */
   template<size_t alignment, typename T = void>
-  inline T* pointer_align_up(void* p)
+  SNMALLOC_FAST_PATH_INLINE T* pointer_align_up(void* p)
   {
     static_assert(alignment > 0);
     static_assert(bits::is_pow2(alignment));
@@ -175,14 +177,15 @@ namespace snmalloc
     size_t alignment,
     typename T = void,
     SNMALLOC_CONCEPT(capptr::IsBound) bounds>
-  inline CapPtr<T, bounds> pointer_align_up(CapPtr<void, bounds> p)
+  SNMALLOC_FAST_PATH_INLINE CapPtr<T, bounds>
+  pointer_align_up(CapPtr<void, bounds> p)
   {
     return CapPtr<T, bounds>::unsafe_from(
       pointer_align_up<alignment, T>(p.unsafe_ptr()));
   }
 
   template<size_t alignment>
-  inline address_t address_align_up(address_t p)
+  SNMALLOC_FAST_PATH_INLINE address_t address_align_up(address_t p)
   {
     return bits::align_up(p, alignment);
   }
@@ -192,7 +195,7 @@ namespace snmalloc
    * a power of two.
    */
   template<typename T = void>
-  inline T* pointer_align_down(void* p, size_t alignment)
+  SNMALLOC_FAST_PATH_INLINE T* pointer_align_down(void* p, size_t alignment)
   {
     SNMALLOC_ASSERT(alignment > 0);
     SNMALLOC_ASSERT(bits::is_pow2(alignment));
@@ -205,7 +208,7 @@ namespace snmalloc
   }
 
   template<typename T = void, SNMALLOC_CONCEPT(capptr::IsBound) bounds>
-  inline CapPtr<T, bounds>
+  SNMALLOC_FAST_PATH_INLINE CapPtr<T, bounds>
   pointer_align_down(CapPtr<void, bounds> p, size_t alignment)
   {
     return CapPtr<T, bounds>::unsafe_from(
@@ -217,7 +220,7 @@ namespace snmalloc
    * be a power of two.
    */
   template<typename T = void>
-  inline T* pointer_align_up(void* p, size_t alignment)
+  SNMALLOC_FAST_PATH_INLINE T* pointer_align_up(void* p, size_t alignment)
   {
     SNMALLOC_ASSERT(alignment > 0);
     SNMALLOC_ASSERT(bits::is_pow2(alignment));
@@ -230,7 +233,7 @@ namespace snmalloc
   }
 
   template<typename T = void, SNMALLOC_CONCEPT(capptr::IsBound) bounds>
-  inline CapPtr<T, bounds>
+  SNMALLOC_FAST_PATH_INLINE CapPtr<T, bounds>
   pointer_align_up(CapPtr<void, bounds> p, size_t alignment)
   {
     return CapPtr<T, bounds>::unsafe_from(
@@ -242,7 +245,8 @@ namespace snmalloc
    * expected to point to the base of some (sub)allocation into which cursor
    * points; would-be negative answers trip an assertion in debug builds.
    */
-  inline size_t pointer_diff(const void* base, const void* cursor)
+  SNMALLOC_FAST_PATH_INLINE size_t
+  pointer_diff(const void* base, const void* cursor)
   {
     SNMALLOC_ASSERT(cursor >= base);
     return static_cast<size_t>(
@@ -254,7 +258,8 @@ namespace snmalloc
     typename U = void,
     SNMALLOC_CONCEPT(capptr::IsBound) Tbounds,
     SNMALLOC_CONCEPT(capptr::IsBound) Ubounds>
-  inline size_t pointer_diff(CapPtr<T, Tbounds> base, CapPtr<U, Ubounds> cursor)
+  SNMALLOC_FAST_PATH_INLINE size_t
+  pointer_diff(CapPtr<T, Tbounds> base, CapPtr<U, Ubounds> cursor)
   {
     return pointer_diff(base.unsafe_ptr(), cursor.unsafe_ptr());
   }
@@ -263,7 +268,8 @@ namespace snmalloc
    * Compute the difference in pointers in units of char. This can be used
    * across allocations.
    */
-  inline ptrdiff_t pointer_diff_signed(void* base, void* cursor)
+  SNMALLOC_FAST_PATH_INLINE ptrdiff_t
+  pointer_diff_signed(void* base, void* cursor)
   {
     return static_cast<ptrdiff_t>(
       static_cast<char*>(cursor) - static_cast<char*>(base));
@@ -274,7 +280,7 @@ namespace snmalloc
     typename U = void,
     SNMALLOC_CONCEPT(capptr::IsBound) Tbounds,
     SNMALLOC_CONCEPT(capptr::IsBound) Ubounds>
-  inline ptrdiff_t
+  SNMALLOC_FAST_PATH_INLINE ptrdiff_t
   pointer_diff_signed(CapPtr<T, Tbounds> base, CapPtr<U, Ubounds> cursor)
   {
     return pointer_diff_signed(base.unsafe_ptr(), cursor.unsafe_ptr());
@@ -285,7 +291,7 @@ namespace snmalloc
    * putative alignment.
    */
   template<size_t alignment>
-  inline size_t address_misalignment(address_t a)
+  SNMALLOC_FAST_PATH_INLINE size_t address_misalignment(address_t a)
   {
     return static_cast<size_t>(a - pointer_align_down<alignment>(a));
   }

diff --git a/src/snmalloc/backend_helpers/range_helpers.h b/src/snmalloc/backend_helpers/range_helpers.h
@@ -16,7 +16,7 @@ namespace snmalloc
 
     // Find the minimum set of maximally aligned blocks in this range.
     // Each block's alignment and size are equal.
-    while (length >= sizeof(void*))
+    while (length >= bits::one_at_bit(MIN_BITS))
     {
       size_t base_align_bits = bits::ctz(address_cast(base));
       size_t length_align_bits = (bits::BITS - 1) - bits::clz(length);

diff --git a/src/snmalloc/mem/freelist.h b/src/snmalloc/mem/freelist.h
@@ -187,6 +187,7 @@ namespace snmalloc
                 signed_prev(address_cast(this), address_cast(n_tame), key));
             }
           }
+          Aal::prefetch(&(n_tame->next_object));
           return n_tame;
         }