Try #672:

libs/pika/async_combinators/include/pika/async_combinators/when_some.hpp

@@ -178,6 +178,7 @@ namespace pika {
 
 # include <pika/config.hpp>
 # include <pika/assert.hpp>
+# include <pika/concurrency/spinlock.hpp>
 # include <pika/functional/deferred_call.hpp>
 # include <pika/futures/future.hpp>
 # include <pika/futures/futures_factory.hpp>
@@ -319,7 +320,7 @@ namespace pika {
         template <typename Sequence>
         struct when_some : std::enable_shared_from_this<when_some<Sequence>>    //-V690
         {
-            using mutex_type = pika::spinlock;
+            using mutex_type = pika::concurrency::detail::spinlock;
 
         public:
             void on_future_ready(std::size_t idx, pika::execution::detail::agent_ref ctx)

libs/pika/async_cuda/src/cuda_event_callback.cpp

@@ -13,10 +13,10 @@
 #include <pika/async_cuda/detail/cuda_debug.hpp>
 #include <pika/async_cuda/detail/cuda_event_callback.hpp>
 #include <pika/concurrency/concurrentqueue.hpp>
+#include <pika/concurrency/spinlock.hpp>
 #include <pika/datastructures/detail/small_vector.hpp>
 #include <pika/resource_partitioner/detail/partitioner.hpp>
 #include <pika/runtime/thread_pool_helpers.hpp>
-#include <pika/synchronization/spinlock.hpp>
 #include <pika/threading_base/scheduler_base.hpp>
 #include <pika/threading_base/thread_pool_base.hpp>
 
@@ -59,7 +59,7 @@ namespace pika::cuda::experimental::detail {
     public:
         using event_callback_queue_type = concurrency::detail::ConcurrentQueue<event_callback>;
         using event_ready_queue_type = concurrency::detail::ConcurrentQueue<ready_callback>;
-        using mutex_type = pika::spinlock;
+        using mutex_type = pika::concurrency::detail::spinlock;
         using event_callback_vector_type = std::vector<event_callback>;
 
         // Background progress function for async CUDA operations. Checks for

libs/pika/async_mpi/src/mpi_polling.cpp

@@ -8,11 +8,11 @@
 #include <pika/assert.hpp>
 #include <pika/async_mpi/mpi_exception.hpp>
 #include <pika/async_mpi/mpi_polling.hpp>
+#include <pika/concurrency/spinlock.hpp>
 #include <pika/modules/errors.hpp>
 #include <pika/modules/threading_base.hpp>
 #include <pika/mpi_base/mpi_environment.hpp>
 #include <pika/synchronization/condition_variable.hpp>
-#include <pika/synchronization/mutex.hpp>
 
 #include <array>
 #include <atomic>
@@ -53,7 +53,7 @@ namespace pika::mpi::experimental {
 
         // -----------------------------------------------------------------
         /// Spinlock is used as it can be called by OS threads or pika tasks
-        using mutex_type = pika::spinlock;
+        using mutex_type = pika::concurrency::detail::spinlock;
 
         // -----------------------------------------------------------------
         /// Queries an environment variable to get/override a default value for

libs/pika/concurrency/include/pika/concurrency/spinlock.hpp

@@ -9,61 +9,114 @@
 #pragma once
 
 #include <pika/config.hpp>
+#include <pika/execution_base/this_thread.hpp>
 #include <pika/lock_registration/detail/register_locks.hpp>
 #include <pika/modules/itt_notify.hpp>
 #include <pika/thread_support/spinlock.hpp>
 
+#include <atomic>
+
 namespace pika::concurrency::detail {
-    // Lockable spinlock class
-    //
-    // This is equivalent to pika::detail::spinlock with the addition of
-    // lock registration.
     struct spinlock
     {
     public:
         PIKA_NON_COPYABLE(spinlock);
 
     private:
-        pika::detail::spinlock m;
+        std::atomic<bool> v_;
 
     public:
-        spinlock(char const* /*desc*/ = nullptr)
+        spinlock(char const* const desc = "pika::concurrency::detail::spinlock")
+          : v_(false)
         {
-            PIKA_ITT_SYNC_CREATE(this, "pika::concurrency::detail::spinlock", "");
+            PIKA_ITT_SYNC_CREATE(this, desc, "");
         }
 
         ~spinlock()
         {
             PIKA_ITT_SYNC_DESTROY(this);
         }
 
-        void lock() noexcept
+        void lock()
         {
             PIKA_ITT_SYNC_PREPARE(this);
-            m.lock();
+
+            // Checking for the value in is_locked() ensures that
+            // acquire_lock is only called when is_locked computes
+            // to false. This way we spin only on a load operation
+            // which minimizes false sharing that comes with an
+            // exchange operation.
+            // Consider the following cases:
+            // 1. Only one thread wants access critical section:
+            //      is_locked() -> false; computes acquire_lock()
+            //      acquire_lock() -> false (new value set to true)
+            //      Thread acquires the lock and moves to critical
+            //      section.
+            // 2. Two threads simultaneously access critical section:
+            //      Thread 1: is_locked() || acquire_lock() -> false
+            //      Thread 1 acquires the lock and moves to critical
+            //      section.
+            //      Thread 2: is_locked() -> true; execution enters
+            //      inside while without computing acquire_lock().
+            //      Thread 2 yields while is_locked() computes to
+            //      false. Then it retries doing is_locked() -> false
+            //      followed by an acquire_lock() operation.
+            //      The above order can be changed arbitrarily but
+            //      the nature of execution will still remain the
+            //      same.
+            do
+            {
+                util::yield_while([this] { return is_locked(); },
+                    "pika::concurrency::detail::spinlock::lock", false);
+            } while (!acquire_lock());
+
             PIKA_ITT_SYNC_ACQUIRED(this);
             util::register_lock(this);
         }
 
-        bool try_lock() noexcept
+        bool try_lock()
         {
             PIKA_ITT_SYNC_PREPARE(this);
-            if (m.try_lock())
+
+            bool r = acquire_lock();    //-V707
+
+            if (r)
             {
                 PIKA_ITT_SYNC_ACQUIRED(this);
                 util::register_lock(this);
                 return true;
             }
+
             PIKA_ITT_SYNC_CANCEL(this);
             return false;
         }
 
-        void unlock() noexcept
+        void unlock()
         {
             PIKA_ITT_SYNC_RELEASING(this);
-            m.unlock();
+
+            relinquish_lock();
+
             PIKA_ITT_SYNC_RELEASED(this);
             util::unregister_lock(this);
         }
+
+    private:
+        // returns whether the mutex has been acquired
+        PIKA_FORCEINLINE bool acquire_lock()
+        {
+            return !v_.exchange(true, std::memory_order_acquire);
+        }
+
+        // relinquish lock
+        PIKA_FORCEINLINE void relinquish_lock()
+        {
+            v_.store(false, std::memory_order_release);
+        }
+
+        PIKA_FORCEINLINE bool is_locked() const
+        {
+            return v_.load(std::memory_order_relaxed);
+        }
     };
 }    // namespace pika::concurrency::detail

libs/pika/errors/include/pika/errors/exception_list.hpp

@@ -34,10 +34,6 @@ namespace pika {
     {
     private:
         /// \cond NOINTERNAL
-
-        // TODO: Does this need to be pika::spinlock?
-        // using mutex_type = pika::spinlock;
-        // TODO: Add correct initialization of pika::util::detail spinlock.
         using mutex_type = pika::detail::spinlock;
 
         using exception_list_type = std::list<std::exception_ptr>;

libs/pika/execution/include/pika/execution/algorithms/ensure_started.hpp

@@ -16,6 +16,7 @@
 # include <pika/allocator_support/traits/is_allocator.hpp>
 # include <pika/assert.hpp>
 # include <pika/concepts/concepts.hpp>
+# include <pika/concurrency/spinlock.hpp>
 # include <pika/datastructures/variant.hpp>
 # include <pika/execution/algorithms/detail/helpers.hpp>
 # include <pika/execution/algorithms/detail/partial_algorithm.hpp>
@@ -27,7 +28,6 @@
 # include <pika/functional/invoke_fused.hpp>
 # include <pika/functional/unique_function.hpp>
 # include <pika/memory/intrusive_ptr.hpp>
-# include <pika/synchronization/spinlock.hpp>
 # include <pika/thread_support/atomic_count.hpp>
 # include <pika/type_support/detail/with_result_of.hpp>
 # include <pika/type_support/pack.hpp>
@@ -123,7 +123,7 @@ namespace pika::ensure_started_detail {
             using allocator_type =
                 typename std::allocator_traits<Allocator>::template rebind_alloc<shared_state>;
             PIKA_NO_UNIQUE_ADDRESS allocator_type alloc;
-            using mutex_type = pika::spinlock;
+            using mutex_type = pika::concurrency::detail::spinlock;
             mutex_type mtx;
             pika::detail::atomic_count reference_count{0};
             std::atomic<bool> start_called{false};

libs/pika/execution/include/pika/execution/algorithms/split.hpp

@@ -16,6 +16,7 @@
 # include <pika/allocator_support/traits/is_allocator.hpp>
 # include <pika/assert.hpp>
 # include <pika/concepts/concepts.hpp>
+# include <pika/concurrency/spinlock.hpp>
 # include <pika/datastructures/detail/small_vector.hpp>
 # include <pika/datastructures/variant.hpp>
 # include <pika/execution/algorithms/detail/helpers.hpp>
@@ -28,7 +29,6 @@
 # include <pika/functional/invoke_fused.hpp>
 # include <pika/functional/unique_function.hpp>
 # include <pika/memory/intrusive_ptr.hpp>
-# include <pika/synchronization/spinlock.hpp>
 # include <pika/thread_support/atomic_count.hpp>
 # include <pika/type_support/detail/with_result_of.hpp>
 # include <pika/type_support/pack.hpp>
@@ -123,7 +123,7 @@ namespace pika::split_detail {
             using allocator_type =
                 typename std::allocator_traits<Allocator>::template rebind_alloc<shared_state>;
             PIKA_NO_UNIQUE_ADDRESS allocator_type alloc;
-            using mutex_type = pika::spinlock;
+            using mutex_type = pika::concurrency::detail::spinlock;
             mutex_type mtx;
             pika::detail::atomic_count reference_count{0};
             std::atomic<bool> start_called{false};

libs/pika/execution/include/pika/execution/algorithms/split_tuple.hpp

@@ -13,6 +13,7 @@
 #include <pika/allocator_support/traits/is_allocator.hpp>
 #include <pika/assert.hpp>
 #include <pika/concepts/concepts.hpp>
+#include <pika/concurrency/spinlock.hpp>
 #include <pika/datastructures/variant.hpp>
 #include <pika/execution/algorithms/detail/helpers.hpp>
 #include <pika/execution/algorithms/detail/partial_algorithm.hpp>
@@ -22,7 +23,6 @@
 #include <pika/functional/detail/tag_fallback_invoke.hpp>
 #include <pika/functional/unique_function.hpp>
 #include <pika/memory/intrusive_ptr.hpp>
-#include <pika/synchronization/spinlock.hpp>
 #include <pika/thread_support/atomic_count.hpp>
 #include <pika/type_support/detail/with_result_of.hpp>
 #include <pika/type_support/pack.hpp>
@@ -59,7 +59,7 @@ namespace pika::split_tuple_detail {
         using allocator_type =
             typename std::allocator_traits<Allocator>::template rebind_alloc<shared_state>;
         PIKA_NO_UNIQUE_ADDRESS allocator_type alloc;
-        using mutex_type = pika::spinlock;
+        using mutex_type = pika::concurrency::detail::spinlock;
         mutex_type mtx;
         pika::detail::atomic_count reference_count{0};
         std::atomic<bool> start_called{false};

libs/pika/execution/include/pika/execution/algorithms/sync_wait.hpp

@@ -13,14 +13,14 @@
 #endif
 
 #include <pika/concepts/concepts.hpp>
+#include <pika/concurrency/spinlock.hpp>
 #include <pika/datastructures/variant.hpp>
 #include <pika/execution/algorithms/detail/helpers.hpp>
 #include <pika/execution_base/operation_state.hpp>
 #include <pika/execution_base/receiver.hpp>
 #include <pika/execution_base/sender.hpp>
 #include <pika/functional/detail/tag_fallback_invoke.hpp>
 #include <pika/synchronization/condition_variable.hpp>
-#include <pika/synchronization/spinlock.hpp>
 #include <pika/type_support/pack.hpp>
 #include <pika/type_support/unused.hpp>
 
@@ -129,7 +129,7 @@ namespace pika::sync_wait_detail {
 #endif
 
         // We use a spinlock here to allow taking the lock on non-pika threads.
-        using mutex_type = pika::spinlock;
+        using mutex_type = pika::concurrency::detail::spinlock;
 
         struct shared_state
         {

libs/pika/executors/include/pika/executors/fork_join_executor.hpp

@@ -11,6 +11,7 @@
 #include <pika/async_base/launch_policy.hpp>
 #include <pika/concurrency/cache_line_data.hpp>
 #include <pika/concurrency/detail/contiguous_index_queue.hpp>
+#include <pika/concurrency/spinlock.hpp>
 #include <pika/coroutines/thread_enums.hpp>
 #include <pika/execution/detail/async_launch_policy_dispatch.hpp>
 #include <pika/execution/executors/execution.hpp>
@@ -20,7 +21,6 @@
 #include <pika/functional/invoke.hpp>
 #include <pika/functional/invoke_fused.hpp>
 #include <pika/modules/itt_notify.hpp>
-#include <pika/synchronization/spinlock.hpp>
 #include <pika/threading/thread.hpp>
 #include <pika/timing/detail/timestamp.hpp>
 
@@ -96,7 +96,7 @@ namespace pika::execution::experimental {
 
             struct region_data_type;
             using thread_function_helper_type = void(region_data_type&, std::size_t, std::size_t,
-                queues_type&, pika::spinlock&, std::exception_ptr&) noexcept;
+                queues_type&, pika::concurrency::detail::spinlock&, std::exception_ptr&) noexcept;
 
             // Members that change for each parallel region.
             struct region_data
@@ -134,7 +134,7 @@ namespace pika::execution::experimental {
 
             std::size_t main_thread_;
             std::size_t num_threads_;
-            pika::spinlock exception_mutex_;
+            pika::concurrency::detail::spinlock exception_mutex_;
             std::exception_ptr exception_;
 
             // Data for each parallel region.
@@ -185,7 +185,7 @@ namespace pika::execution::experimental {
                 std::size_t const num_threads_;
                 std::size_t const thread_index_;
                 loop_schedule const schedule_;
-                pika::spinlock& exception_mutex_;
+                pika::concurrency::detail::spinlock& exception_mutex_;
                 std::exception_ptr& exception_;
                 std::uint64_t yield_delay_;
 
@@ -366,7 +366,8 @@ namespace pika::execution::experimental {
                 /// Main entry point for a single parallel region (static
                 /// scheduling).
                 static void call_static(region_data_type& rdata, std::size_t thread_index,
-                    std::size_t num_threads, queues_type&, pika::spinlock& exception_mutex,
+                    std::size_t num_threads, queues_type&,
+                    pika::concurrency::detail::spinlock& exception_mutex,
                     std::exception_ptr& exception) noexcept
                 {
                     region_data& data = rdata[thread_index].data_;
@@ -410,7 +411,8 @@ namespace pika::execution::experimental {
                 /// Main entry point for a single parallel region (dynamic
                 /// scheduling).
                 static void call_dynamic(region_data_type& rdata, std::size_t thread_index,
-                    std::size_t num_threads, queues_type& queues, pika::spinlock& exception_mutex,
+                    std::size_t num_threads, queues_type& queues,
+                    pika::concurrency::detail::spinlock& exception_mutex,
                     std::exception_ptr& exception) noexcept
                 {
                     region_data& data = rdata[thread_index].data_;

libs/pika/futures/include/pika/futures/detail/future_data.hpp

@@ -9,6 +9,7 @@
 #include <pika/config.hpp>
 #include <pika/assert.hpp>
 #include <pika/async_base/launch_policy.hpp>
+#include <pika/concurrency/spinlock.hpp>
 #include <pika/coroutines/detail/get_stack_pointer.hpp>
 #include <pika/datastructures/detail/small_vector.hpp>
 #include <pika/errors/try_catch_exception_ptr.hpp>
@@ -18,7 +19,6 @@
 #include <pika/memory/intrusive_ptr.hpp>
 #include <pika/modules/errors.hpp>
 #include <pika/synchronization/condition_variable.hpp>
-#include <pika/synchronization/spinlock.hpp>
 #include <pika/thread_support/assert_owns_lock.hpp>
 #include <pika/thread_support/atomic_count.hpp>
 #include <pika/threading_base/annotated_function.hpp>
@@ -205,7 +205,7 @@ namespace pika::lcos::detail {
     template <>
     struct PIKA_EXPORT future_data_base<traits::detail::future_data_void> : future_data_refcnt_base
     {
-        using mutex_type = pika::spinlock;
+        using mutex_type = pika::concurrency::detail::spinlock;
 
         future_data_base() noexcept
           : state_(empty)