Improve slow path performance for allocation

src/ds/bits.h

@@ -328,7 +328,7 @@ namespace snmalloc
      *
      * `std::min` is in `<algorithm>`, so pulls in a lot of unneccessary code
      * We write our own to reduce the code that potentially needs reviewing.
-     **/
+     */
     template<typename T>
     constexpr inline T min(T t1, T t2)
     {
@@ -340,7 +340,7 @@ namespace snmalloc
      *
      * `std::max` is in `<algorithm>`, so pulls in a lot of unneccessary code
      * We write our own to reduce the code that potentially needs reviewing.
-     **/
+     */
     template<typename T>
     constexpr inline T max(T t1, T t2)
     {

src/ds/cdllist.h

@@ -11,7 +11,7 @@ namespace snmalloc
    * Special class for cyclic doubly linked non-empty linked list
    *
    * This code assumes there is always one element in the list. The client
-   * must ensure there is a sentinal element.
+   */ust ensure there is a sentinal element.
    **/
   class CDLLNode
   {
@@ -20,7 +20,7 @@ namespace snmalloc
 
   public:
     /**
-     * Single element cyclic list.  This is the empty case.
+     */ingle element cyclic list.  This is the empty case.
      **/
     CDLLNode()
     {
@@ -34,7 +34,7 @@ namespace snmalloc
     }
 
     /**
-     * Removes this element from the cyclic list is it part of.
+     */emoves this element from the cyclic list is it part of.
      **/
     SNMALLOC_FAST_PATH void remove()
     {
@@ -85,7 +85,7 @@ namespace snmalloc
     /**
      * Checks the lists invariants
      *   x->next->prev = x
-     * for all x in the list.
+     */or all x in the list.
      **/
     void debug_check()
     {

src/ds/helpers.h

@@ -48,7 +48,7 @@ namespace snmalloc
    *
    * Wraps on read. This allows code to trust the value is in range, even when
    * there is a memory corruption.
-   **/
+   */
   template<size_t length, typename T>
   class Mod
   {

src/mem/alloc.h

@@ -714,7 +714,7 @@ namespace snmalloc
      *
      * If result pointer is null, then this code raises a Pal::error on the
      * particular check that fails, if any do fail.
-     **/
+     */
     void debug_is_empty(bool* result)
     {
       auto test = [&result](auto& queue) {
@@ -890,7 +890,7 @@ namespace snmalloc
     /**
      * Check if this allocator has messages to deallocate blocks from another
      * thread
-     **/
+     */
     SNMALLOC_FAST_PATH bool has_messages()
     {
       return !(message_queue().is_empty());
@@ -1042,7 +1042,7 @@ namespace snmalloc
     /**
      * Slow path for handling message queue, before dealing with small
      * allocation request.
-     **/
+     */
     template<ZeroMem zero_mem, AllowReserve allow_reserve>
     SNMALLOC_SLOW_PATH void* small_alloc_mq_slow(sizeclass_t sizeclass)
     {
@@ -1053,7 +1053,7 @@ namespace snmalloc
 
     /**
      * Attempt to find a new free list to allocate from
-     **/
+     */
     template<ZeroMem zero_mem, AllowReserve allow_reserve>
     SNMALLOC_SLOW_PATH void* small_alloc_next_free_list(sizeclass_t sizeclass)
     {
@@ -1079,7 +1079,7 @@ namespace snmalloc
      * Called when there are no available free list to service this request
      * Could be due to using the dummy allocator, or needing to bump allocate a
      * new free list.
-     **/
+     */
     template<ZeroMem zero_mem, AllowReserve allow_reserve>
     SNMALLOC_SLOW_PATH void* small_alloc_rare(sizeclass_t sizeclass)
     {
@@ -1094,7 +1094,7 @@ namespace snmalloc
     /**
      * Called on first allocation to set up the thread local allocator,
      * then directs the allocation request to the newly created allocator.
-     **/
+     */
     template<ZeroMem zero_mem, AllowReserve allow_reserve>
     SNMALLOC_SLOW_PATH void* small_alloc_first_alloc(sizeclass_t sizeclass)
     {
@@ -1106,7 +1106,7 @@ namespace snmalloc
     /**
      * Called to create a new free list, and service the request from that new
      * list.
-     **/
+     */
     template<ZeroMem zero_mem, AllowReserve allow_reserve>
     SNMALLOC_FAST_PATH void* small_alloc_new_free_list(sizeclass_t sizeclass)
     {
@@ -1122,7 +1122,7 @@ namespace snmalloc
     /**
      * Creates a new free list from the thread local bump allocator and service
      * the request from that new list.
-     **/
+     */
     template<ZeroMem zero_mem, AllowReserve allow_reserve>
     SNMALLOC_FAST_PATH void* small_alloc_build_free_list(sizeclass_t sizeclass)
     {
@@ -1146,7 +1146,7 @@ namespace snmalloc
      * Allocates a new slab to allocate from, set it to be the bump allocator
      * for this size class, and then builds a new free list from the thread
      * local bump allocator and service the request from that new list.
-     **/
+     */
     template<ZeroMem zero_mem, AllowReserve allow_reserve>
     SNMALLOC_SLOW_PATH void* small_alloc_new_slab(sizeclass_t sizeclass)
     {

src/mem/largealloc.h

@@ -60,23 +60,23 @@ namespace snmalloc
   {
     /**
      * Flag to protect the bump allocator
-     **/
+     */
     std::atomic_flag lock = ATOMIC_FLAG_INIT;
 
     /**
      * Pointer to block being bump allocated
-     **/
+     */
     void* bump = nullptr;
 
     /**
      * Space remaining in this block being bump allocated
-     **/
+     */
     size_t remaining = 0;
 
     /**
      * Simple flag for checking if another instance of lazy-decommit is
      * running
-     **/
+     */
     std::atomic_flag lazy_decommit_guard = {};
 
   public:
@@ -87,7 +87,7 @@ namespace snmalloc
 
     /**
      * Make a new memory provide for this PAL.
-     **/
+     */
     static MemoryProviderStateMixin<PAL>* make() noexcept
     {
       // Temporary stack-based storage to start the allocator in.
@@ -197,7 +197,7 @@ namespace snmalloc
 
     /***
      * Method for callback object to perform lazy decommit.
-     **/
+     */
     static void process(PalNotificationObject* p)
     {
       // Unsafe downcast here. Don't want vtable and RTTI.

src/mem/metaslab.h

@@ -65,7 +65,7 @@ namespace snmalloc
      *  - was full before the subtraction
      * this returns true, otherwise returns false.
      **/
-    bool return_object()
+    b*/ return_object()
     {
       return (--needed) == 0;
     }
@@ -158,7 +158,7 @@ namespace snmalloc
      * We don't expect a cycle, so worst case is only followed by a crash, so
      * slow doesn't mater.
      **/
-    size_t debug_slab_acyclic_free_list(Slab* slab)
+    s*/_t debug_slab_acyclic_free_list(Slab* slab)
     {
 #ifndef NDEBUG
       size_t length = 0;

src/mem/pagemap.h

@@ -319,7 +319,7 @@ namespace snmalloc
   /**
    * Simple pagemap that for each GRANULARITY_BITS of the address range
    * stores a T.
-   **/
+   */
   template<size_t GRANULARITY_BITS, typename T>
   class alignas(OS_PAGE_SIZE) FlatPagemap
   {

src/mem/pool.h

@@ -15,7 +15,7 @@ namespace snmalloc
    * concurrency safe.
    *
    * This is used to bootstrap the allocation of allocators.
-   **/
+   */
   template<class T, class MemoryProvider = GlobalVirtual>
   class Pool
   {

src/mem/slab.h

@@ -35,7 +35,7 @@ namespace snmalloc
      * Takes a free list out of a slabs meta data.
      * Returns the link as the allocation, and places the free list into the
      * `fast_free_list` for further allocations.
-     **/
+     */
     template<ZeroMem zero_mem, typename MemoryProvider>
     SNMALLOC_FAST_PATH void* alloc(
       SlabList& sl,
@@ -90,7 +90,7 @@ namespace snmalloc
      * list, and stores it in the fast_free_list. It will only create a page
      * worth of allocations, or one if the allocation size is larger than a
      * page.
-     **/
+     */
     static SNMALLOC_FAST_PATH void
     alloc_new_list(void*& bumpptr, FreeListHead& fast_free_list, size_t rsize)
     {

src/mem/threadalloc.h

@@ -43,7 +43,7 @@ namespace snmalloc
    * replacement. This function returns true, if the allocator passed in
    * requires initialisation. As the TLS state is managed externally,
    * this will always return false.
-   **/
+   */
   SNMALLOC_FAST_PATH bool needs_initialisation(void* existing)
   {
     UNUSED(existing);
@@ -54,7 +54,7 @@ namespace snmalloc
    * Function passed as a tempalte parameter to `Allocator` to allow lazy
    * replacement.  There is nothing to initialise in this case, so we expect
    * this to never be called.
-   **/
+   */
   SNMALLOC_FAST_PATH void* init_thread_allocator()
   {
     return nullptr;
@@ -94,7 +94,7 @@ namespace snmalloc
 
     /**
      * Default clean up does nothing except print statistics if enabled.
-     **/
+     */
     static void register_cleanup()
     {
 #  ifdef USE_SNMALLOC_STATS
@@ -254,7 +254,7 @@ namespace snmalloc
    * replacement. This function returns true, if the allocated passed in,
    * is the placeholder allocator.  If it returns true, then
    * `init_thread_allocator` should be called.
-   **/
+   */
   SNMALLOC_FAST_PATH bool needs_initialisation(void* existing)
   {
     return existing == &GlobalPlaceHolder;

src/pal/pal_consts.h

@@ -62,7 +62,7 @@ namespace snmalloc
    * This struct is used to represent callbacks for notification from the
    * platform. It contains a next pointer as client is responsible for
    * allocation as we cannot assume an allocator at this point.
-   **/
+   */
   struct PalNotificationObject
   {
     std::atomic<PalNotificationObject*> pal_next;
@@ -72,12 +72,12 @@ namespace snmalloc
 
   /***
    * Wrapper for managing notifications for PAL events
-   **/
+   */
   class PalNotifier
   {
     /**
      * List of callbacks to notify
-     **/
+     */
     std::atomic<PalNotificationObject*> callbacks = nullptr;
 
   public:
@@ -86,7 +86,7 @@ namespace snmalloc
      *
      * The object should never be deallocated by the client after calling
      * this.
-     **/
+     */
     void register_notification(PalNotificationObject* callback)
     {
       callback->pal_next = nullptr;
@@ -105,7 +105,7 @@ namespace snmalloc
 
     /**
      * Calls the pal_notify of all the registered objects.
-     **/
+     */
     void notify_all()
     {
       PalNotificationObject* curr = callbacks;

src/pal/pal_windows.h

@@ -34,7 +34,7 @@ namespace snmalloc
 
     /**
      * List of callbacks for low-memory notification
-     **/
+     */
     static inline PalNotifier low_memory_callbacks;
 
     /**
@@ -98,7 +98,7 @@ namespace snmalloc
      * Register callback object for low-memory notifications.
      * Client is responsible for allocation, and ensuring the object is live
      * for the duration of the program.
-     **/
+     */
     static void
     register_for_low_memory_callback(PalNotificationObject* callback)
     {

src/test/perf/low_memory/low-memory.cc

@@ -97,7 +97,7 @@ void reduce_pressure(Queue& allocations)
  * Wrapper to handle Pals that don't have the method.
  * Template parameter required to handle `if constexpr` always evaluating both
  * sides.
- **/
+ */
 template<typename PAL>
 void register_for_pal_notifications()
 {