From bfc4414a959b1a7cfeaf184431aba2b4b1dca93c Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Sun, 2 Feb 2020 21:57:32 -0800 Subject: [PATCH 01/91] sync: adds Notify for basic task notification `Notify` provides a synchronization primitive similar to thread park / unpark, except for tasks. --- tokio/src/loom/std/mod.rs | 1 + tokio/src/macros/pin.rs | 10 +- tokio/src/sync/mod.rs | 12 + tokio/src/sync/notify.rs | 496 ++++++++++++++++++++++++++++ tokio/src/sync/tests/loom_notify.rs | 88 +++++ tokio/src/sync/tests/mod.rs | 1 + tokio/src/util/linked_list.rs | 483 +++++++++++++++++++++++++++ tokio/src/util/mod.rs | 2 + tokio/tests/sync_notify.rs | 102 ++++++ 9 files changed, 1194 insertions(+), 1 deletion(-) create mode 100644 tokio/src/sync/notify.rs create mode 100644 tokio/src/sync/tests/loom_notify.rs create mode 100644 tokio/src/util/linked_list.rs create mode 100644 tokio/tests/sync_notify.rs diff --git a/tokio/src/loom/std/mod.rs b/tokio/src/loom/std/mod.rs index d5e057e51e8..e4bae357b5f 100644 --- a/tokio/src/loom/std/mod.rs +++ b/tokio/src/loom/std/mod.rs @@ -64,6 +64,7 @@ pub(crate) mod sync { pub(crate) use crate::loom::std::atomic_u64::AtomicU64; pub(crate) use crate::loom::std::atomic_usize::AtomicUsize; + pub(crate) use std::sync::atomic::AtomicU8; pub(crate) use std::sync::atomic::{fence, AtomicPtr}; pub(crate) use std::sync::atomic::{spin_loop_hint, AtomicBool}; } diff --git a/tokio/src/macros/pin.rs b/tokio/src/macros/pin.rs index e8511d3f3dc..f3dc66e710c 100644 --- a/tokio/src/macros/pin.rs +++ b/tokio/src/macros/pin.rs @@ -108,5 +108,13 @@ macro_rules! pin { let mut $x = unsafe { $crate::macros::support::Pin::new_unchecked(&mut $x) }; - )* } + )* }; + ($( + let $x:ident = $init:expr; + )*) => { + $( + let $x = $init; + crate::pin!($x); + )* + } } diff --git a/tokio/src/sync/mod.rs b/tokio/src/sync/mod.rs index ff72c521360..befdcae8eb9 100644 --- a/tokio/src/sync/mod.rs +++ b/tokio/src/sync/mod.rs @@ -406,9 +406,18 @@ //! * [`Mutex`][Mutex] Mutual Exclusion mechanism, which ensures that at most //! one thread at a time is able to access some data. //! +//! * [`Notify`][Notify] Basic task notification. `Notify` supports notifying a +//! receiving task without sending data. In this case, the task wakes up and +//! resumes processing. +//! //! * [`RwLock`][RwLock] Provides a mutual exclusion mechanism which allows //! multiple readers at the same time, while allowing only one writer at a //! time. In some cases, this can be more efficient than a mutex. +//! +//! * [`Semaphore`][Semaphore] Limits the amount of concurrency. A semaphore +//! holds a number of permits, which tasks may request in order to enter a +//! critical section. Semaphores are useful for implementing limiting of +//! bounding of any kind. cfg_sync! { mod barrier; @@ -421,6 +430,9 @@ cfg_sync! { mod mutex; pub use mutex::{Mutex, MutexGuard}; + mod notify; + pub use notify::Notify; + pub mod oneshot; pub(crate) mod semaphore_ll; diff --git a/tokio/src/sync/notify.rs b/tokio/src/sync/notify.rs new file mode 100644 index 00000000000..b0b25c425ec --- /dev/null +++ b/tokio/src/sync/notify.rs @@ -0,0 +1,496 @@ +use crate::loom::sync::atomic::AtomicU8; +use crate::loom::sync::Mutex; +use crate::util::linked_list::{self, LinkedList}; + +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::Ordering::SeqCst; +use std::task::{Context, Poll, Waker}; + +/// Notify a single task to wake up. +/// +/// `Notify` provides a basic mechanism to notify a single task of an event. +/// `Notify` itself does not carry any data. Instead, it is to be used to signal +/// another task to perform an operation. +/// +/// The synchronization details of `Notify` are similar to +/// [`thread::park`][park] and [`Thread::unpark`][unpark] from std. A [`Notify`] +/// value contains a single permit. [`Notify::recv`] waits for the permit to be +/// made available, consumes the permit, and resumes. [`Notify::notify_one`] +/// sets the permit, waking a pending task if there is one. +/// +/// `Notify` can be thought of as a [`Semaphore`] starting with 0 permits. +/// [`Notify::recv`] waits for a permit to become available and +/// [`Notify::notify_one`] sets a permit **if there currently are no available +/// permits**. +/// +/// If `notify_one` is called **before** `recv()`, then the next call to +/// `recv()` will complete immediately, consuming the permit. Any subsequent +/// calls to `recv()` will wait for a new permit. +/// +/// If `notify_one` is called **multiple** times before `recv()`, only a +/// **single** permit is stored. The next call to `recv()` will complete +/// immediately, but the one after will wait for a new permit. +/// +/// # Examples +/// +/// Basic usage. +/// +/// ``` +/// use tokio::sync::Notify; +/// use std::sync::Arc; +/// +/// #[tokio::main] +/// async fn main() { +/// let notify = Arc::new(Notify::new()); +/// let notify2 = notify.clone(); +/// +/// tokio::spawn(async move { +/// notify2.recv().await; +/// println!("received a notification"); +/// }); +/// +/// notify.notify_one(); +/// } +/// ``` +/// +/// Unbound mpsc channel. +/// +/// ``` +/// use tokio::sync::Notify; +/// use std::collections::VecDeque; +/// use std::sync::Mutex; +/// +/// struct Channel { +/// values: Mutex>, +/// notify: Notify, +/// } +/// +/// impl Channel { +/// pub fn send(&self, value: T) { +/// self.values.lock().unwrap() +/// .push_back(value); +/// +/// // Notify the consumer a value is available +/// self.notify.notify_one(); +/// } +/// +/// pub async fn recv(&self) -> T { +/// loop { +/// // Drain values +/// if let Some(value) = self.values.lock().unwrap().pop_front() { +/// return value; +/// } +/// +/// // Wait for values to be available +/// self.notify.recv().await; +/// } +/// } +/// } +/// +/// ``` +/// +/// [park]: std::thread::park +/// [unpark]: std::thread::Thread::unpark +/// [`Notify::recv`]: Notify::recv() +/// [`Notify::notify_one`]: Notify::notify_one() +/// [`Semaphore`]: crate::sync::Semaphore +#[derive(Debug)] +pub struct Notify { + state: AtomicU8, + waiters: Mutex>, +} + +#[derive(Debug)] +struct Waiter { + /// Waiting task's waker + waker: Option, + + /// `true` if the notification has been assigned to this waiter. + notified: bool, +} + +#[derive(Debug)] +struct RecvFuture<'a> { + /// The `Notify` being received on. + notify: &'a Notify, + + /// The current state of the receiving process. + state: RecvState, + + /// Entry in the waiter `LinkedList`. + waiter: linked_list::Entry, +} + +#[derive(Debug)] +enum RecvState { + Init, + Waiting, + Done, +} + +/// Initial "idle" state +const EMPTY: u8 = 0; + +/// One or more threads are currently waiting to be notified. +const WAITING: u8 = 1; + +/// Pending notification +const NOTIFIED: u8 = 2; + +impl Notify { + /// Create a new `Notify`, initialized without a permit. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::Notify; + /// + /// let notify = Notify::new(); + /// ``` + pub fn new() -> Notify { + Notify { + state: AtomicU8::new(0), + waiters: Mutex::new(LinkedList::new()), + } + } + + /// Wait for a notification. + /// + /// Each `Notify` value holds a single permit. If a permit is available from + /// an earlier call to `notify_one`, then `recv` will complete immediately, + /// consuming that permit. Otherwise, `recv()` waits for a permit to be made + /// available by the next call to `notify_one` + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::Notify; + /// use std::sync::Arc; + /// + /// #[tokio::main] + /// async fn main() { + /// let notify = Arc::new(Notify::new()); + /// let notify2 = notify.clone(); + /// + /// tokio::spawn(async move { + /// notify2.recv().await; + /// println!("received a notification"); + /// }); + /// + /// notify.notify_one(); + /// } + /// ``` + pub async fn recv(&self) { + RecvFuture { + notify: self, + state: RecvState::Init, + waiter: linked_list::Entry::new(Waiter { + waker: None, + notified: false, + }), + } + .await + } + + /// Notifies a waiting task + /// + /// If a task is currently waiting, that task is notified. Otherwise, a + /// permit is stored in this `Notify` value and the **next** call to + /// `recv()` will complete immediately consuming the permit made available + /// by this call to `notify_one()`. + /// + /// At most one permit may be stored by `Notify`. Many sequential calls to + /// `notify_one` will result in a single permit being stored. The next call + /// to `recv()` will complete immediately, but the one after that will wait. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::Notify; + /// use std::sync::Arc; + /// + /// #[tokio::main] + /// async fn main() { + /// let notify = Arc::new(Notify::new()); + /// let notify2 = notify.clone(); + /// + /// tokio::spawn(async move { + /// notify2.recv().await; + /// println!("received a notification"); + /// }); + /// + /// notify.notify_one(); + /// } + /// ``` + pub fn notify_one(&self) { + // Load the current state + let mut curr = self.state.load(SeqCst); + + // If the state is `EMPTY`, transition to `NOTIFIED` and return. + loop { + match curr { + EMPTY | NOTIFIED => { + // The compare-exchange from `NOTIFIED` -> `NOTIFIED` is + // intended a happens-before synchronization must happen + // between this atomic operation and a task calling + // `recv()`. + let res = self.state.compare_exchange(curr, NOTIFIED, SeqCst, SeqCst); + + match res { + // No waiters, no further work to do + Ok(_) => return, + Err(actual) => { + curr = actual; + } + } + } + _ => break, + } + } + + // There are waiters, the lock must be acquired to notify. + let mut waiters = self.waiters.lock().unwrap(); + + // The state must be reloaded while the lock is held. The state may only + // transition out of WAITING while the lock is held. + curr = self.state.load(SeqCst); + + if let Some(waker) = notify_locked(&mut waiters, &self.state, curr) { + drop(waiters); + waker.wake(); + } + } +} + +fn notify_locked( + waiters: &mut LinkedList, + state: &AtomicU8, + mut curr: u8, +) -> Option { + loop { + match curr { + EMPTY | NOTIFIED => { + let res = state.compare_exchange(curr, NOTIFIED, SeqCst, SeqCst); + + match res { + Ok(_) => return None, + Err(actual) => { + curr = actual; + } + } + } + WAITING => { + // At this point, it is guaranteed that the state will not + // concurrently change as holding the lock is required to + // transition **out** of `WAITING`. + // + // Get a pending waiter + let mut waiter = waiters.pop_back().unwrap(); + + assert!(!waiter.notified); + + waiter.notified = true; + let waker = waiter.waker.take(); + + if waiters.is_empty() { + // As this the **final** waiter in the list, the state + // must be transitioned to `EMPTY`. As transitioning + // **from** `WAITING` requires the lock to be held, a + // `store` is sufficient. + state.store(EMPTY, SeqCst); + } + + return waker; + } + _ => unreachable!(), + } + } +} + +// ===== impl RecvFuture ===== + +impl RecvFuture<'_> { + fn project( + self: Pin<&mut Self>, + ) -> ( + &Notify, + &mut RecvState, + Pin<&mut linked_list::Entry>, + ) { + unsafe { + // Safety: both `notify` and `state` are `Unpin`. + + is_unpin::<&Notify>(); + is_unpin::(); + + let me = self.get_unchecked_mut(); + ( + &me.notify, + &mut me.state, + Pin::new_unchecked(&mut me.waiter), + ) + } + } +} + +impl Future for RecvFuture<'_> { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + use RecvState::*; + + let (notify, state, mut waiter) = self.project(); + + 'outer: loop { + match *state { + Init => { + // Optimistically try acquiring a pending notification + let res = notify + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst); + + if res.is_ok() { + // Acquired the notification + *state = Done; + continue 'outer; + } + + // Acquire the lock and attempt to transition to the waiting + // state. + let mut waiters = notify.waiters.lock().unwrap(); + + // Reload the state with the lock held + let mut curr = notify.state.load(SeqCst); + + // Transition the state to WAITING. + loop { + match curr { + EMPTY => { + // Transition to WAITING + let res = notify + .state + .compare_exchange(EMPTY, WAITING, SeqCst, SeqCst); + + if let Err(actual) = res { + curr = actual; + continue; + } + } + WAITING => {} + NOTIFIED => { + // Try consuming the notification + let res = notify + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst); + + match res { + Ok(_) => { + // Acquired the notification + *state = Done; + continue 'outer; + } + Err(actual) => { + curr = actual; + continue; + } + } + } + _ => unreachable!(), + } + + // The state has been transitioned to waiting + break; + } + + // Safety: called while locked. + unsafe { + (*waiter.as_mut().get()).waker = Some(cx.waker().clone()); + + // Insert the waiter into the linked list + waiters.push_front(waiter.as_mut()); + } + + *state = Waiting; + } + Waiting => { + let mut waiters = notify.waiters.lock().unwrap(); + + { + // Safety: called while locked + let w = unsafe { &mut *waiter.as_mut().get() }; + + if w.notified { + w.waker = None; + w.notified = false; + + // Remove the entry from the linked list. The entry + // may only be in the linked list while the state is + // `Waiting`. + // + // Safety: called while the lock is held + unsafe { + waiters.remove(waiter.as_mut()); + } + + *state = Done; + } else { + if !w.waker.as_ref().unwrap().will_wake(cx.waker()) { + w.waker = Some(cx.waker().clone()); + } + + return Poll::Pending; + } + } + } + Done => { + return Poll::Ready(()); + } + } + } + } +} + +impl Drop for RecvFuture<'_> { + fn drop(&mut self) { + use RecvState::*; + + // Safety: The type only transitions to a "Waiting" state when pinned. + let (notify, state, mut waiter) = unsafe { Pin::new_unchecked(self).project() }; + + // This is where we ensure safety. The `RecvFuture` is being dropped, + // which means we must ensure that the waiter entry is no longer stored + // in the linked list. + if let Waiting = *state { + let mut notify_state = WAITING; + let mut waiters = notify.waiters.lock().unwrap(); + + // remove the entry from the list + // + // safety: the waiter is only added to `waiters` by virtue of it + // being the only `LinkedList` available to the type. + unsafe { waiters.remove(waiter.as_mut()) }; + + if waiters.is_empty() { + notify_state = EMPTY; + notify.state.store(EMPTY, SeqCst); + } + + // See if the node was notified but not received. In this case, the + // notification must be sent to another waiter. + // + // Safety: with the entry removed from the linked list, there can be + // no concurrent access to the entry + let notified = unsafe { (*waiter.as_mut().get()).notified }; + + if notified { + if let Some(waker) = notify_locked(&mut waiters, ¬ify.state, notify_state) { + drop(waiters); + waker.wake(); + } + } + } + } +} + +fn is_unpin() {} diff --git a/tokio/src/sync/tests/loom_notify.rs b/tokio/src/sync/tests/loom_notify.rs new file mode 100644 index 00000000000..23fc2f35aa7 --- /dev/null +++ b/tokio/src/sync/tests/loom_notify.rs @@ -0,0 +1,88 @@ +use crate::sync::Notify; + +use loom::future::block_on; +use loom::sync::Arc; +use loom::thread; + +#[test] +fn notify_one() { + loom::model(|| { + let tx = Arc::new(Notify::new()); + let rx = tx.clone(); + + let th = thread::spawn(move || { + block_on(async { + rx.recv().await; + }); + }); + + tx.notify_one(); + th.join().unwrap(); + }); +} + +#[test] +fn notify_multi() { + loom::model(|| { + let notify = Arc::new(Notify::new()); + + let mut ths = vec![]; + + for _ in 0..2 { + let notify = notify.clone(); + + ths.push(thread::spawn(move || { + block_on(async { + notify.recv().await; + notify.notify_one(); + }) + })); + } + + notify.notify_one(); + + for th in ths.drain(..) { + th.join().unwrap(); + } + + block_on(async { + notify.recv().await; + }); + }); +} + +#[test] +fn notify_drop() { + use crate::future::poll_fn; + use std::future::Future; + use std::task::Poll; + + loom::model(|| { + let notify = Arc::new(Notify::new()); + let rx1 = notify.clone(); + let rx2 = notify.clone(); + + let th1 = thread::spawn(move || { + let mut recv = Box::pin(async { + rx1.recv().await; + rx1.notify_one(); + }); + + block_on(poll_fn(|cx| { + let _ = recv.as_mut().poll(cx); + Poll::Ready(()) + })); + }); + + let th2 = thread::spawn(move || { + block_on(async { + rx2.recv().await; + }); + }); + + notify.notify_one(); + + th1.join().unwrap(); + th2.join().unwrap(); + }); +} diff --git a/tokio/src/sync/tests/mod.rs b/tokio/src/sync/tests/mod.rs index 2ee140cbf18..7225ce9c58c 100644 --- a/tokio/src/sync/tests/mod.rs +++ b/tokio/src/sync/tests/mod.rs @@ -8,6 +8,7 @@ cfg_loom! { mod loom_broadcast; mod loom_list; mod loom_mpsc; + mod loom_notify; mod loom_oneshot; mod loom_semaphore_ll; } diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs new file mode 100644 index 00000000000..1d50bdc4899 --- /dev/null +++ b/tokio/src/util/linked_list.rs @@ -0,0 +1,483 @@ +//! An intrusive double linked list of data +//! +//! The data structure supports tracking pinned nodes. Most of the data +//! structure's APIs are `unsafe` as they require the caller to ensure the +//! specified node is actually + +use core::cell::UnsafeCell; +use core::marker::PhantomPinned; +use core::pin::Pin; +use core::ptr::NonNull; + +/// An intrusive linked list of nodes, where each node carries associated data +/// of type `T`. +#[derive(Debug)] +pub(crate) struct LinkedList { + head: Option>>, + tail: Option>>, +} + +unsafe impl Send for LinkedList {} +unsafe impl Sync for LinkedList {} + +/// A node which carries data of type `T` and is stored in an intrusive list. +#[derive(Debug)] +pub(crate) struct Entry { + /// The previous node in the list. null if there is no previous node. + prev: Option>>, + + /// The next node in the list. null if there is no previous node. + next: Option>>, + + /// The data which is associated to this list item + data: UnsafeCell, + + /// Prevents `Entry`s from being `Unpin`. They may never be moved, since + /// the list semantics require addresses to be stable. + _pin: PhantomPinned, +} + +unsafe impl Send for Entry {} +unsafe impl Sync for Entry {} + +impl LinkedList { + /// Creates an empty linked list + pub(crate) fn new() -> Self { + LinkedList { + head: None, + tail: None, + } + } + + /// Adds an item to the back of the linked list. + /// + /// # Safety + /// + /// The function is only safe as long as valid pointers are stored inside + /// the linked list. + pub(crate) unsafe fn push_front(&mut self, entry: Pin<&mut Entry>) { + let mut entry: NonNull> = entry.get_unchecked_mut().into(); + + entry.as_mut().next = self.head; + entry.as_mut().prev = None; + + if let Some(head) = &mut self.head { + head.as_mut().prev = Some(entry); + } + + self.head = Some(entry); + + if self.tail.is_none() { + self.tail = Some(entry); + } + } + + /// Removes the first element and returns it, or `None` if the list is empty. + /// + /// The function is safe as the lifetime of the entry is bound to `&mut + /// self`. + pub(crate) fn pop_back(&mut self) -> Option> { + unsafe { + let mut last = match self.tail { + Some(tail) => { + self.tail = tail.as_ref().prev; + tail + } + None => return None, + }; + + if let Some(mut prev) = last.as_mut().prev { + prev.as_mut().next = None; + } else { + self.head = None + } + + last.as_mut().prev = None; + last.as_mut().next = None; + + let val = &mut *last.as_mut().data.get(); + + Some(Pin::new_unchecked(val)) + } + } + + /// Returns whether the linked list doesn not contain any node + pub(crate) fn is_empty(&self) -> bool { + if !self.head.is_none() { + return false; + } + + assert!(self.tail.is_none()); + true + } + + /// Removes the given item from the linked list. + /// + /// # Safety + /// + /// The caller **must** ensure that `entry` is currently contained by + /// `self`. + pub(crate) unsafe fn remove(&mut self, entry: Pin<&mut Entry>) -> bool { + let mut entry = NonNull::from(entry.get_unchecked_mut()); + + if let Some(mut prev) = entry.as_mut().prev { + debug_assert_eq!(prev.as_ref().next, Some(entry)); + prev.as_mut().next = entry.as_ref().next; + } else { + if self.head != Some(entry) { + return false; + } + + self.head = entry.as_ref().next; + } + + if let Some(mut next) = entry.as_mut().next { + debug_assert_eq!(next.as_ref().prev, Some(entry)); + next.as_mut().prev = entry.as_ref().prev; + } else { + // This might be the last item in the list + if self.tail != Some(entry) { + return false; + } + + self.tail = entry.as_ref().prev; + } + + entry.as_mut().next = None; + entry.as_mut().prev = None; + + true + } +} + +impl Entry { + /// Creates a new node with the associated data + pub(crate) fn new(data: T) -> Entry { + Entry { + prev: None, + next: None, + data: UnsafeCell::new(data), + _pin: PhantomPinned, + } + } + + /// Get a raw pointer to the inner data + pub(crate) fn get(&self) -> *mut T { + self.data.get() + } +} + +#[cfg(test)] +#[cfg(not(loom))] +mod tests { + use super::*; + + fn collect_list(list: &mut LinkedList) -> Vec { + let mut ret = vec![]; + + while let Some(v) = list.pop_back() { + ret.push(*v); + } + + ret + } + + unsafe fn push_all(list: &mut LinkedList, entries: &mut [Pin<&mut Entry>]) { + for entry in entries.iter_mut() { + list.push_front(entry.as_mut()); + } + } + + macro_rules! assert_clean { + ($e:ident) => {{ + assert!($e.next.is_none()); + assert!($e.prev.is_none()); + }}; + } + + macro_rules! assert_ptr_eq { + ($a:expr, $b:expr) => {{ + // Deal with mapping a Pin<&mut T> -> Option> + assert_eq!(Some($a.as_mut().get_unchecked_mut().into()), $b) + }}; + } + + #[test] + fn push_and_drain() { + pin! { + let a = Entry::new(5); + let b = Entry::new(7); + let c = Entry::new(31); + } + + let mut list = LinkedList::new(); + assert!(list.is_empty()); + + unsafe { + list.push_front(a); + assert!(!list.is_empty()); + list.push_front(b); + list.push_front(c); + } + + let items: Vec = collect_list(&mut list); + assert_eq!([5, 7, 31].to_vec(), items); + + assert!(list.is_empty()); + } + + #[test] + fn push_pop_push_pop() { + pin! { + let a = Entry::new(5); + let b = Entry::new(7); + } + + let mut list = LinkedList::new(); + + unsafe { + list.push_front(a); + } + + let v = list.pop_back().unwrap(); + assert_eq!(5, *v); + assert!(list.is_empty()); + + unsafe { + list.push_front(b); + } + + let v = list.pop_back().unwrap(); + assert_eq!(7, *v); + + assert!(list.is_empty()); + assert!(list.pop_back().is_none()); + } + + #[test] + fn remove_by_address() { + pin! { + let a = Entry::new(5); + let b = Entry::new(7); + let c = Entry::new(31); + } + + unsafe { + // Remove first + let mut list = LinkedList::new(); + + push_all(&mut list, &mut [c.as_mut(), b.as_mut(), a.as_mut()]); + assert!(list.remove(a.as_mut())); + assert_clean!(a); + // `a` should be no longer there and can't be removed twice + assert!(!list.remove(a.as_mut())); + assert!(!list.is_empty()); + + assert!(list.remove(b.as_mut())); + assert_clean!(b); + // `b` should be no longer there and can't be removed twice + assert!(!list.remove(b.as_mut())); + assert!(!list.is_empty()); + + assert!(list.remove(c.as_mut())); + assert_clean!(c); + // `b` should be no longer there and can't be removed twice + assert!(!list.remove(c.as_mut())); + assert!(list.is_empty()); + } + + unsafe { + // Remove middle + let mut list = LinkedList::new(); + + push_all(&mut list, &mut [c.as_mut(), b.as_mut(), a.as_mut()]); + + assert!(list.remove(a.as_mut())); + assert_clean!(a); + + assert_ptr_eq!(b, list.head); + assert_ptr_eq!(c, b.next); + assert_ptr_eq!(b, c.prev); + + let items = collect_list(&mut list); + assert_eq!([31, 7].to_vec(), items); + } + + unsafe { + // Remove middle + let mut list = LinkedList::new(); + + push_all(&mut list, &mut [c.as_mut(), b.as_mut(), a.as_mut()]); + + assert!(list.remove(b.as_mut())); + assert_clean!(b); + + assert_ptr_eq!(c, a.next); + assert_ptr_eq!(a, c.prev); + + let items = collect_list(&mut list); + assert_eq!([31, 5].to_vec(), items); + } + + unsafe { + // Remove last + // Remove middle + let mut list = LinkedList::new(); + + push_all(&mut list, &mut [c.as_mut(), b.as_mut(), a.as_mut()]); + + assert!(list.remove(c.as_mut())); + assert_clean!(c); + + assert!(b.next.is_none()); + assert_ptr_eq!(b, list.tail); + + let items = collect_list(&mut list); + assert_eq!([7, 5].to_vec(), items); + } + + unsafe { + // Remove first of two + let mut list = LinkedList::new(); + + push_all(&mut list, &mut [b.as_mut(), a.as_mut()]); + + assert!(list.remove(a.as_mut())); + + assert_clean!(a); + + // a should be no longer there and can't be removed twice + assert!(!list.remove(a.as_mut())); + + assert_ptr_eq!(b, list.head); + assert_ptr_eq!(b, list.tail); + + assert!(b.next.is_none()); + assert!(b.prev.is_none()); + + let items = collect_list(&mut list); + assert_eq!([7].to_vec(), items); + } + + unsafe { + // Remove last of two + let mut list = LinkedList::new(); + + push_all(&mut list, &mut [b.as_mut(), a.as_mut()]); + + assert!(list.remove(b.as_mut())); + + assert_clean!(b); + + assert_ptr_eq!(a, list.head); + assert_ptr_eq!(a, list.tail); + + assert!(a.next.is_none()); + assert!(a.prev.is_none()); + + let items = collect_list(&mut list); + assert_eq!([5].to_vec(), items); + } + + unsafe { + // Remove last item + let mut list = LinkedList::new(); + + push_all(&mut list, &mut [a.as_mut()]); + + assert!(list.remove(a.as_mut())); + assert_clean!(a); + + assert!(list.head.is_none()); + assert!(list.tail.is_none()); + let items = collect_list(&mut list); + assert!(items.is_empty()); + } + + unsafe { + // Remove missing + let mut list = LinkedList::new(); + + list.push_front(b.as_mut()); + list.push_front(a.as_mut()); + + assert!(!list.remove(c.as_mut())); + } + } + + proptest::proptest! { + #[test] + fn fuzz_linked_list(ops: Vec) { + run_fuzz(ops); + } + } + + fn run_fuzz(ops: Vec) { + use std::collections::VecDeque; + + #[derive(Debug)] + enum Op { + Push, + Pop, + Remove(usize), + } + + let ops = ops + .iter() + .map(|i| match i % 3 { + 0 => Op::Push, + 1 => Op::Pop, + 2 => Op::Remove(i / 3), + _ => unreachable!(), + }) + .collect::>(); + + let mut next = 0; + let mut ll = LinkedList::new(); + let mut entries = VecDeque::new(); + let mut reference = VecDeque::new(); + + for op in ops { + match op { + Op::Push => { + let v = next; + next += 1; + + reference.push_front(v); + entries.push_front(Box::pin(Entry::new(v))); + + unsafe { + ll.push_front(entries.front_mut().unwrap().as_mut()); + } + } + Op::Pop => { + if reference.is_empty() { + assert!(ll.is_empty()); + continue; + } + + let v = reference.pop_back(); + assert_eq!(v, ll.pop_back().map(|v| *v)); + entries.pop_back(); + } + Op::Remove(n) => { + if reference.is_empty() { + assert!(ll.is_empty()); + continue; + } + + let idx = n % reference.len(); + + unsafe { + assert!(ll.remove(entries[idx].as_mut())); + } + + let v = reference.remove(idx).unwrap(); + assert_eq!(v, unsafe { *entries[idx].get() }); + + entries.remove(idx); + } + } + } + } +} diff --git a/tokio/src/util/mod.rs b/tokio/src/util/mod.rs index cd5b151d137..ad8e1b0d097 100644 --- a/tokio/src/util/mod.rs +++ b/tokio/src/util/mod.rs @@ -3,6 +3,8 @@ cfg_io_driver! { pub(crate) mod slab; } +pub(crate) mod linked_list; + #[cfg(any(feature = "rt-threaded", feature = "macros", feature = "stream"))] mod rand; diff --git a/tokio/tests/sync_notify.rs b/tokio/tests/sync_notify.rs new file mode 100644 index 00000000000..6f0e4cfdb9e --- /dev/null +++ b/tokio/tests/sync_notify.rs @@ -0,0 +1,102 @@ +#![warn(rust_2018_idioms)] +#![cfg(feature = "full")] + +use tokio::sync::Notify; +use tokio_test::task::spawn; +use tokio_test::*; + +trait AssertSend: Send + Sync {} +impl AssertSend for Notify {} + +#[test] +fn notify_recv_one() { + let notify = Notify::new(); + let mut recv = spawn(async { notify.recv().await }); + + notify.notify_one(); + assert_ready!(recv.poll()); +} + +#[test] +fn recv_one_notify() { + let notify = Notify::new(); + let mut recv = spawn(async { notify.recv().await }); + + assert_pending!(recv.poll()); + + notify.notify_one(); + assert!(recv.is_woken()); + assert_ready!(recv.poll()); +} + +#[test] +fn recv_multi_notify() { + let notify = Notify::new(); + let mut recv1 = spawn(async { notify.recv().await }); + let mut recv2 = spawn(async { notify.recv().await }); + + assert_pending!(recv1.poll()); + assert_pending!(recv2.poll()); + + notify.notify_one(); + assert!(recv1.is_woken()); + assert!(!recv2.is_woken()); + + assert_ready!(recv1.poll()); + assert_pending!(recv2.poll()); +} + +#[test] +fn notify_recv_multi() { + let notify = Notify::new(); + + notify.notify_one(); + + let mut recv1 = spawn(async { notify.recv().await }); + let mut recv2 = spawn(async { notify.recv().await }); + + assert_ready!(recv1.poll()); + assert_pending!(recv2.poll()); + + notify.notify_one(); + + assert!(recv2.is_woken()); + assert_ready!(recv2.poll()); +} + +#[test] +fn recv_drop_recv_notify() { + let notify = Notify::new(); + let mut recv1 = spawn(async { notify.recv().await }); + let mut recv2 = spawn(async { notify.recv().await }); + + assert_pending!(recv1.poll()); + + drop(recv1); + + assert_pending!(recv2.poll()); + + notify.notify_one(); + assert!(recv2.is_woken()); + assert_ready!(recv2.poll()); +} + +#[test] +fn recv_multi_notify_drop_one() { + let notify = Notify::new(); + let mut recv1 = spawn(async { notify.recv().await }); + let mut recv2 = spawn(async { notify.recv().await }); + + assert_pending!(recv1.poll()); + assert_pending!(recv2.poll()); + + notify.notify_one(); + + assert!(recv1.is_woken()); + assert!(!recv2.is_woken()); + + drop(recv1); + + assert!(recv2.is_woken()); + assert_ready!(recv2.poll()); +} From 18906454273d77cc0faa2c736b9e0079eee3f583 Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Mon, 3 Feb 2020 09:05:04 -0800 Subject: [PATCH 02/91] fix warnings --- tokio/src/sync/notify.rs | 36 +++++++++++++++++------------------ tokio/src/util/linked_list.rs | 2 +- tokio/src/util/mod.rs | 4 +++- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/tokio/src/sync/notify.rs b/tokio/src/sync/notify.rs index b0b25c425ec..e761bb5b2d3 100644 --- a/tokio/src/sync/notify.rs +++ b/tokio/src/sync/notify.rs @@ -87,7 +87,6 @@ use std::task::{Context, Poll, Waker}; /// } /// } /// } -/// /// ``` /// /// [park]: std::thread::park @@ -228,24 +227,19 @@ impl Notify { let mut curr = self.state.load(SeqCst); // If the state is `EMPTY`, transition to `NOTIFIED` and return. - loop { - match curr { - EMPTY | NOTIFIED => { - // The compare-exchange from `NOTIFIED` -> `NOTIFIED` is - // intended a happens-before synchronization must happen - // between this atomic operation and a task calling - // `recv()`. - let res = self.state.compare_exchange(curr, NOTIFIED, SeqCst, SeqCst); - - match res { - // No waiters, no further work to do - Ok(_) => return, - Err(actual) => { - curr = actual; - } - } + while let EMPTY | NOTIFIED = curr { + // The compare-exchange from `NOTIFIED` -> `NOTIFIED` is + // intended a happens-before synchronization must happen + // between this atomic operation and a task calling + // `recv()`. + let res = self.state.compare_exchange(curr, NOTIFIED, SeqCst, SeqCst); + + match res { + // No waiters, no further work to do + Ok(_) => return, + Err(actual) => { + curr = actual; } - _ => break, } } @@ -263,6 +257,12 @@ impl Notify { } } +impl Default for Notify { + fn default() -> Notify { + Notify::new() + } +} + fn notify_locked( waiters: &mut LinkedList, state: &AtomicU8, diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 1d50bdc4899..b2486069317 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -103,7 +103,7 @@ impl LinkedList { /// Returns whether the linked list doesn not contain any node pub(crate) fn is_empty(&self) -> bool { - if !self.head.is_none() { + if self.head.is_some() { return false; } diff --git a/tokio/src/util/mod.rs b/tokio/src/util/mod.rs index ad8e1b0d097..2761f7252fd 100644 --- a/tokio/src/util/mod.rs +++ b/tokio/src/util/mod.rs @@ -3,7 +3,9 @@ cfg_io_driver! { pub(crate) mod slab; } -pub(crate) mod linked_list; +cfg_sync! { + pub(crate) mod linked_list; +} #[cfg(any(feature = "rt-threaded", feature = "macros", feature = "stream"))] mod rand; From 5e82767e11af828fbf47be3ef0e5e32582421b13 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 17 Feb 2020 14:41:41 -0800 Subject: [PATCH 03/91] wip Signed-off-by: Eliza Weisman --- tokio/src/sync/mod.rs | 1 + tokio/src/sync/semaphore_ll2.rs | 112 ++++++++++++++++++++++++++++++++ tokio/src/util/linked_list.rs | 4 ++ 3 files changed, 117 insertions(+) create mode 100644 tokio/src/sync/semaphore_ll2.rs diff --git a/tokio/src/sync/mod.rs b/tokio/src/sync/mod.rs index befdcae8eb9..0f6a2fced10 100644 --- a/tokio/src/sync/mod.rs +++ b/tokio/src/sync/mod.rs @@ -435,6 +435,7 @@ cfg_sync! { pub mod oneshot; + pub(crate) mod semaphore_ll2; pub(crate) mod semaphore_ll; mod semaphore; pub use semaphore::{Semaphore, SemaphorePermit}; diff --git a/tokio/src/sync/semaphore_ll2.rs b/tokio/src/sync/semaphore_ll2.rs new file mode 100644 index 00000000000..c91d88287c4 --- /dev/null +++ b/tokio/src/sync/semaphore_ll2.rs @@ -0,0 +1,112 @@ +use crate::loom::{ + alloc, + future::AtomicWaker, + sync::{atomic::AtomicUsize, Mutex}, +}; +use crate::util::linked_list::{self, LinkedList}; + +use std::{ + pin::Pin, + ptr::NonNull, + sync::atomic::Ordering, + task::{ + Context, Poll, + Poll::{Pending, Ready}, + }, +}; + +pub(crate) struct Semaphore { + waiters: Mutex>>, + permits: AtomicUsize, +} + +/// Error returned by `Permit::poll_acquire`. +#[derive(Debug)] +pub(crate) struct AcquireError(()); + +pin_project_lite::pin_project! { + pub(crate) struct Permit { + #[pin] + node: WaitNode, + state: PermitState, + } +} + +type WaitNode = linked_list::Entry>; + +/// Permit state +#[derive(Debug, Copy, Clone)] +enum PermitState { + /// Currently waiting for permits to be made available and assigned to the + /// waiter. + Waiting(u16), + + /// The number of acquired permits + Acquired(u16), +} + +struct Waiter { + waker: AtomicWaker, + needed: u16, +} + +const CLOSED: usize = std::usize::MAX; + +impl Semaphore { + /// Creates a new semaphore with the initial number of permits + /// + /// # Panics + /// + /// Panics if `permits` is zero. + pub(crate) fn new(permits: usize) -> Self { + assert!(permits > 0, "number of permits must be greater than 0"); + Self { + permits: AtomicUsize::new(permits), + waiters: Mutex::new(LinkedList::new()), + } + } + /// Returns the current number of available permits + pub(crate) fn available_permits(&self) -> usize { + self.permits.load(Ordering::Acquire) + } + + fn poll_acquire( + &self, + cx: &mut Context<'_>, + needed: u16, + node: Pin<&mut WaitNode>, + ) -> Poll> { + let mut curr = self.permits.load(Ordering::Acquire); + let remaining = loop { + if curr == CLOSED { + return Ready(Err(AcquireError(()))); + } + let next = curr.saturating_sub(needed as usize); + match self.permits.compare_exchange_weak( + curr, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) if curr >= needed as usize => return Ready(Ok(())), + Ok(_) => break needed - curr as u16, + Err(actual) => curr = actual, + } + }; + // if we've not returned already, then we need to push the waiter to the + // wait queue. + assert!(node.is_unlinked()); + { + let mut waiter = unsafe { &mut *node.get() }.get_mut(); + waiter.needed = remaining; + waiter.waker.register_by_ref(cx.waker()); + } + + let mut queue = self.waiters.lock().unwrap(); + unsafe { + queue.push_front(node); + } + + Pending + } +} diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index b2486069317..ff5060be8c6 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -165,6 +165,10 @@ impl Entry { pub(crate) fn get(&self) -> *mut T { self.data.get() } + + pub(crate) fn is_unlinked(&self) -> bool { + self.prev.is_none() && self.next.is_none() + } } #[cfg(test)] From ea8ba09376522609848d9337b33cdc76ec8359e6 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 17 Feb 2020 16:31:06 -0800 Subject: [PATCH 04/91] wip2 Signed-off-by: Eliza Weisman --- tokio/src/sync/semaphore_ll2.rs | 218 ++++++++++++++++++++++++++++++++ tokio/src/util/linked_list.rs | 11 ++ 2 files changed, 229 insertions(+) diff --git a/tokio/src/sync/semaphore_ll2.rs b/tokio/src/sync/semaphore_ll2.rs index c91d88287c4..a2d51462bbd 100644 --- a/tokio/src/sync/semaphore_ll2.rs +++ b/tokio/src/sync/semaphore_ll2.rs @@ -18,6 +18,7 @@ use std::{ pub(crate) struct Semaphore { waiters: Mutex>>, permits: AtomicUsize, + add_lock: AtomicUsize, } /// Error returned by `Permit::poll_acquire`. @@ -63,13 +64,94 @@ impl Semaphore { Self { permits: AtomicUsize::new(permits), waiters: Mutex::new(LinkedList::new()), + add_lock: AtomicUsize::new(0), } } + /// Returns the current number of available permits pub(crate) fn available_permits(&self) -> usize { self.permits.load(Ordering::Acquire) } + /// Closes the semaphore. This prevents the semaphore from issuing new + /// permits and notifies all pending waiters. + pub(crate) fn close(&self) { + // Acquire the `add_lock`, setting the "closed" flag on the lock. + let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); + + if prev != 0 { + // Another thread has the lock and will be responsible for notifying + // pending waiters. + return; + } + + self.add_permits_locked(0, true); + } + + /// Adds `n` new permits to the semaphore. + pub(crate) fn add_permits(&self, n: usize) { + if n == 0 { + return; + } + + // TODO: Handle overflow. A panic is not sufficient, the process must + // abort. + let prev = self.add_lock.fetch_add(n << 1, Ordering::AcqRel); + if prev != 0 { + // Another thread has the lock and will be responsible for notifying + // pending waiters. + return; + } + self.add_permits_locked(n, false); + } + + fn add_permits_locked(&self, mut rem: usize, mut closed: bool) { + while rem > 0 || closed { + if closed { + SemState::fetch_set_closed(&self.state, AcqRel); + } + + // Release the permits and notify + let mut waiters = self.waiters.lock().unwrap(); + while rem > 0 { + let pop = { + let mut last = match waiters.last_mut() { + Some(mut last) => last, + None => { + self.permits.fetch_add(rem, Ordering::Release); + break; + } + }; + if rem >= last.needed { + rem -= last.needed; + last.needed = 0; + true + } else { + last.neede -= rem; + break; + } + }; + if pop { + waiters.pop_back().unwrap().get_mut().waker.wake(); + } + } + + let n = rem << 1; + + let actual = if closed { + let actual = self.add_lock.fetch_sub(n | 1, AcqRel); + closed = false; + actual + } else { + let actual = self.add_lock.fetch_sub(n, AcqRel); + closed = actual & 1 == 1; + actual + }; + + rem = (actual >> 1) - rem; + } + } + fn poll_acquire( &self, cx: &mut Context<'_>, @@ -110,3 +192,139 @@ impl Semaphore { Pending } } + +impl Permit { + /// Creates a new `Permit`. + /// + /// The permit begins in the "unacquired" state. + pub(crate) fn new() -> Permit { + use PermitState::Acquired; + + Permit { + node: linked_list::Entry::new(alloc::Track::new(Waiter { + waker: AtomicWaker::new(), + needed: 0, + })), + state: Acquired(0), + } + } + + /// Returns `true` if the permit has been acquired + pub(crate) fn is_acquired(&self) -> bool { + match self.state { + PermitState::Acquired(num) if num > 0 => true, + _ => false, + } + } + + /// Tries to acquire the permit. If no permits are available, the current task + /// is notified once a new permit becomes available. + pub(crate) fn poll_acquire( + &mut self, + cx: &mut Context<'_>, + num_permits: u16, + semaphore: &Semaphore, + ) -> Poll> { + use std::cmp::Ordering::*; + use PermitState::*; + + match self.state { + Waiting(requested) => { + // There must be a waiter + let waiter = self.waiter.as_ref().unwrap(); + + match requested.cmp(&num_permits) { + Less => { + let delta = num_permits - requested; + + // Request additional permits. If the waiter has been + // dequeued, it must be re-queued. + if !waiter.try_inc_permits_to_acquire(delta as usize) { + let waiter = NonNull::from(&**waiter); + + // Ignore the result. The check for + // `permits_to_acquire()` will converge the state as + // needed + let _ = semaphore.poll_acquire2(delta, || Some(waiter))?; + } + + self.state = Waiting(num_permits); + } + Greater => { + let delta = requested - num_permits; + let to_release = waiter.try_dec_permits_to_acquire(delta as usize); + + semaphore.add_permits(to_release); + self.state = Waiting(num_permits); + } + Equal => {} + } + + if waiter.permits_to_acquire()? == 0 { + self.state = Acquired(requested); + return Ready(Ok(())); + } + + waiter.waker.register_by_ref(cx.waker()); + + if waiter.permits_to_acquire()? == 0 { + self.state = Acquired(requested); + return Ready(Ok(())); + } + + Pending + } + Acquired(acquired) => { + if acquired >= num_permits { + Ready(Ok(())) + } else { + match semaphore.poll_acquire(cx, num_permits - acquired, self)? { + Ready(()) => { + self.state = Acquired(num_permits); + Ready(Ok(())) + } + Pending => { + self.state = Waiting(num_permits); + Pending + } + } + } + } + } + } + + /// Tries to acquire the permit. + pub(crate) fn try_acquire( + &mut self, + num_permits: u16, + semaphore: &Semaphore, + ) -> Result<(), TryAcquireError> { + unimplemented!() + } + + /// Releases a permit back to the semaphore + pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { + let n = self.forget(n); + semaphore.add_permits(n as usize); + } + + /// Forgets the permit **without** releasing it back to the semaphore. + /// + /// After calling `forget`, `poll_acquire` is able to acquire new permit + /// from the sempahore. + /// + /// Repeatedly calling `forget` without associated calls to `add_permit` + /// will result in the semaphore losing all permits. + /// + /// Will forget **at most** the number of acquired permits. This number is + /// returned. + pub(crate) fn forget(&mut self, n: u16) -> u16 { + unimplemented!() + } +} + +impl Default for Permit { + fn default() -> Self { + Self::new() + } +} diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index ff5060be8c6..0ee6b5adee7 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -101,6 +101,17 @@ impl LinkedList { } } + /// Borrows the last element and returns it, or `None` if the list is empty. + /// + /// The function is safe as the lifetime of the entry is bound to `&self`. + pub(crate) fn last_mut(&self) -> Option> { + unsafe { + let last = self.tail.as_ref()?.as_ref(); + let val = &mut *last.data.get(); + Some(Pin::new_unchecked(val)) + } + } + /// Returns whether the linked list doesn not contain any node pub(crate) fn is_empty(&self) -> bool { if self.head.is_some() { From a888cb7a045ca0affa41cd868237591bbb193684 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 18 Feb 2020 16:52:13 -0800 Subject: [PATCH 05/91] let's see just how broken it is (it's really broken) Signed-off-by: Eliza Weisman --- tokio/src/sync/mod.rs | 2 +- tokio/src/sync/mutex.rs | 14 +- tokio/src/sync/semaphore.rs | 14 +- tokio/src/sync/semaphore_ll.rs | 1133 ++++---------------------- tokio/src/sync/semaphore_ll2.rs | 330 -------- tokio/src/sync/semaphore_ll_old.rs | 1219 ++++++++++++++++++++++++++++ tokio/src/util/linked_list.rs | 4 +- 7 files changed, 1392 insertions(+), 1324 deletions(-) delete mode 100644 tokio/src/sync/semaphore_ll2.rs create mode 100644 tokio/src/sync/semaphore_ll_old.rs diff --git a/tokio/src/sync/mod.rs b/tokio/src/sync/mod.rs index 0f6a2fced10..c197ef03f15 100644 --- a/tokio/src/sync/mod.rs +++ b/tokio/src/sync/mod.rs @@ -435,7 +435,7 @@ cfg_sync! { pub mod oneshot; - pub(crate) mod semaphore_ll2; + // pub(crate) mod semaphore_ll2; pub(crate) mod semaphore_ll; mod semaphore; pub use semaphore::{Semaphore, SemaphorePermit}; diff --git a/tokio/src/sync/mutex.rs b/tokio/src/sync/mutex.rs index c625ce274cf..2b1e8a51544 100644 --- a/tokio/src/sync/mutex.rs +++ b/tokio/src/sync/mutex.rs @@ -152,18 +152,20 @@ impl Mutex { /// A future that resolves on acquiring the lock and returns the `MutexGuard`. pub async fn lock(&self) -> MutexGuard<'_, T> { - let mut guard = MutexGuard { - lock: self, - permit: semaphore::Permit::new(), - }; - poll_fn(|cx| guard.permit.poll_acquire(cx, 1, &self.s)) + let permit = semaphore::Permit::new(); + let pinned = permit; + pin!(pinned); + poll_fn(|cx| pinned.poll_acquire(cx, 1, &self.s)) .await .unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and we have a // handle to it through the Arc, which means that this can never happen. unreachable!() }); - guard + MutexGuard { + lock: self, + permit, + } } /// Tries to acquire the lock diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index 7721e01f52c..a3eec95d80e 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -56,14 +56,16 @@ impl Semaphore { /// Acquires permit from the semaphore pub async fn acquire(&self) -> SemaphorePermit<'_> { - let mut permit = SemaphorePermit { - sem: &self, - ll_permit: ll::Permit::new(), - }; - poll_fn(|cx| permit.ll_permit.poll_acquire(cx, 1, &self.ll_sem)) + let mut ll_permit = ll::Permit::new(); + let pinned = ll_permit; + pin!(pinned); + poll_fn(|cx| pinned.poll_acquire(cx, 1, &self.ll_sem)) .await .unwrap(); - permit + SemaphorePermit { + sem: &self, + ll_permit, + } } /// Tries to acquire a permit form the semaphore diff --git a/tokio/src/sync/semaphore_ll.rs b/tokio/src/sync/semaphore_ll.rs index 69fd4a6a5d3..28dea85a7f5 100644 --- a/tokio/src/sync/semaphore_ll.rs +++ b/tokio/src/sync/semaphore_ll.rs @@ -1,100 +1,47 @@ -#![cfg_attr(not(feature = "sync"), allow(dead_code, unreachable_pub))] +use crate::loom::{ + alloc, + future::AtomicWaker, + sync::{atomic::AtomicUsize, Mutex}, +}; +use crate::util::linked_list::{self, LinkedList}; + +use std::{ + cmp, + fmt, + pin::Pin, + ptr::NonNull, + sync::atomic::Ordering, + task::{ + Context, Poll, + Poll::{Pending, Ready}, + }, +}; -//! Thread-safe, asynchronous counting semaphore. -//! -//! A `Semaphore` instance holds a set of permits. Permits are used to -//! synchronize access to a shared resource. -//! -//! Before accessing the shared resource, callers acquire a permit from the -//! semaphore. Once the permit is acquired, the caller then enters the critical -//! section. If no permits are available, then acquiring the semaphore returns -//! `Pending`. The task is woken once a permit becomes available. - -use crate::loom::cell::CausalCell; -use crate::loom::future::AtomicWaker; -use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize}; -use crate::loom::thread; - -use std::cmp; -use std::fmt; -use std::ptr::{self, NonNull}; -use std::sync::atomic::Ordering::{self, AcqRel, Acquire, Relaxed, Release}; -use std::task::Poll::{Pending, Ready}; -use std::task::{Context, Poll}; -use std::usize; - -/// Futures-aware semaphore. pub(crate) struct Semaphore { - /// Tracks both the waiter queue tail pointer and the number of remaining - /// permits. - state: AtomicUsize, - - /// waiter queue head pointer. - head: CausalCell>, - - /// Coordinates access to the queue head. - rx_lock: AtomicUsize, - - /// Stub waiter node used as part of the MPSC channel algorithm. - stub: Box, + waiters: Mutex>, + permits: AtomicUsize, + add_lock: AtomicUsize, } -/// A semaphore permit -/// -/// Tracks the lifecycle of a semaphore permit. -/// -/// An instance of `Permit` is intended to be used with a **single** instance of -/// `Semaphore`. Using a single instance of `Permit` with multiple semaphore -/// instances will result in unexpected behavior. -/// -/// `Permit` does **not** release the permit back to the semaphore on drop. It -/// is the user's responsibility to ensure that `Permit::release` is called -/// before dropping the permit. -#[derive(Debug)] -pub(crate) struct Permit { - waiter: Option>, - state: PermitState, -} - -/// Error returned by `Permit::poll_acquire`. -#[derive(Debug)] -pub(crate) struct AcquireError(()); - /// Error returned by `Permit::try_acquire`. #[derive(Debug)] pub(crate) enum TryAcquireError { Closed, NoPermits, } - -/// Node used to notify the semaphore waiter when permit is available. +/// Error returned by `Permit::poll_acquire`. #[derive(Debug)] -struct Waiter { - /// Stores waiter state. - /// - /// See `WaiterState` for more details. - state: AtomicUsize, - - /// Task to wake when a permit is made available. - waker: AtomicWaker, +pub(crate) struct AcquireError(()); - /// Next pointer in the queue of waiting senders. - next: AtomicPtr, +pin_project_lite::pin_project! { + #[derive(Debug)] + pub(crate) struct Permit { + #[pin] + node: linked_list::Entry, + state: PermitState, + } } -/// Semaphore state -/// -/// The 2 low bits track the modes. -/// -/// - Closed -/// - Full -/// -/// When not full, the rest of the `usize` tracks the total number of messages -/// in the channel. When full, the rest of the `usize` is a pointer to the tail -/// of the "waiting senders" queue. -#[derive(Copy, Clone)] -struct SemState(usize); - /// Permit state #[derive(Debug, Copy, Clone)] enum PermitState { @@ -106,47 +53,12 @@ enum PermitState { Acquired(u16), } -/// State for an individual waker node -#[derive(Debug, Copy, Clone)] -struct WaiterState(usize); - -/// Waiter node is in the semaphore queue -const QUEUED: usize = 0b001; - -/// Semaphore has been closed, no more permits will be issued. -const CLOSED: usize = 0b10; - -/// The permit that owns the `Waiter` dropped. -const DROPPED: usize = 0b100; - -/// Represents "one requested permit" in the waiter state -const PERMIT_ONE: usize = 0b1000; - -/// Masks the waiter state to only contain bits tracking number of requested -/// permits. -const PERMIT_MASK: usize = usize::MAX - (PERMIT_ONE - 1); - -/// How much to shift a permit count to pack it into the waker state -const PERMIT_SHIFT: u32 = PERMIT_ONE.trailing_zeros(); - -/// Flag differentiating between available permits and waiter pointers. -/// -/// If we assume pointers are properly aligned, then the least significant bit -/// will always be zero. So, we use that bit to track if the value represents a -/// number. -const NUM_FLAG: usize = 0b01; - -/// Signal the semaphore is closed -const CLOSED_FLAG: usize = 0b10; - -/// Maximum number of permits a semaphore can manage -const MAX_PERMITS: usize = usize::MAX >> NUM_SHIFT; - -/// When representing "numbers", the state has to be shifted this much (to get -/// rid of the flag bit). -const NUM_SHIFT: usize = 2; +struct Waiter { + waker: AtomicWaker, + state: AtomicUsize, +} -// ===== impl Semaphore ===== +const CLOSED: usize = std::usize::MAX; impl Semaphore { /// Creates a new semaphore with the initial number of permits @@ -154,176 +66,25 @@ impl Semaphore { /// # Panics /// /// Panics if `permits` is zero. - pub(crate) fn new(permits: usize) -> Semaphore { - let stub = Box::new(Waiter::new()); - let ptr = NonNull::from(&*stub); - - // Allocations are aligned - debug_assert!(ptr.as_ptr() as usize & NUM_FLAG == 0); - - let state = SemState::new(permits, &stub); - - Semaphore { - state: AtomicUsize::new(state.to_usize()), - head: CausalCell::new(ptr), - rx_lock: AtomicUsize::new(0), - stub, + pub(crate) fn new(permits: usize) -> Self { + assert!(permits > 0, "number of permits must be greater than 0"); + Self { + permits: AtomicUsize::new(permits), + waiters: Mutex::new(LinkedList::new()), + add_lock: AtomicUsize::new(0), } } /// Returns the current number of available permits pub(crate) fn available_permits(&self) -> usize { - let curr = SemState(self.state.load(Acquire)); - curr.available_permits() - } - - /// Tries to acquire the requested number of permits, registering the waiter - /// if not enough permits are available. - fn poll_acquire( - &self, - cx: &mut Context<'_>, - num_permits: u16, - permit: &mut Permit, - ) -> Poll> { - self.poll_acquire2(num_permits, || { - let waiter = permit.waiter.get_or_insert_with(|| Box::new(Waiter::new())); - - waiter.waker.register_by_ref(cx.waker()); - - Some(NonNull::from(&**waiter)) - }) - } - - fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { - match self.poll_acquire2(num_permits, || None) { - Poll::Ready(res) => res.map_err(to_try_acquire), - Poll::Pending => Err(TryAcquireError::NoPermits), - } - } - - /// Polls for a permit - /// - /// Tries to acquire available permits first. If unable to acquire a - /// sufficient number of permits, the caller's waiter is pushed onto the - /// semaphore's wait queue. - fn poll_acquire2( - &self, - num_permits: u16, - mut get_waiter: F, - ) -> Poll> - where - F: FnMut() -> Option>, - { - let num_permits = num_permits as usize; - - // Load the current state - let mut curr = SemState(self.state.load(Acquire)); - - // Saves a ref to the waiter node - let mut maybe_waiter: Option> = None; - - /// Used in branches where we attempt to push the waiter into the wait - /// queue but fail due to permits becoming available or the wait queue - /// transitioning to "closed". In this case, the waiter must be - /// transitioned back to the "idle" state. - macro_rules! revert_to_idle { - () => { - if let Some(waiter) = maybe_waiter { - unsafe { waiter.as_ref() }.revert_to_idle(); - } - }; - } - - loop { - let mut next = curr; - - if curr.is_closed() { - revert_to_idle!(); - return Ready(Err(AcquireError::closed())); - } - - let acquired = next.acquire_permits(num_permits, &self.stub); - - if !acquired { - // There are not enough available permits to satisfy the - // request. The permit transitions to a waiting state. - debug_assert!(curr.waiter().is_some() || curr.available_permits() < num_permits); - - if let Some(waiter) = maybe_waiter.as_ref() { - // Safety: the caller owns the waiter. - let w = unsafe { waiter.as_ref() }; - w.set_permits_to_acquire(num_permits - curr.available_permits()); - } else { - // Get the waiter for the permit. - if let Some(waiter) = get_waiter() { - // Safety: the caller owns the waiter. - let w = unsafe { waiter.as_ref() }; - - // If there are any currently available permits, the - // waiter acquires those immediately and waits for the - // remaining permits to become available. - if !w.to_queued(num_permits - curr.available_permits()) { - // The node is alrady queued, there is no further work - // to do. - return Pending; - } - - maybe_waiter = Some(waiter); - } else { - // No waiter, this indicates the caller does not wish to - // "wait", so there is nothing left to do. - return Pending; - } - } - - next.set_waiter(maybe_waiter.unwrap()); - } - - debug_assert_ne!(curr.0, 0); - debug_assert_ne!(next.0, 0); - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => { - if acquired { - // Successfully acquire permits **without** queuing the - // waiter node. The waiter node is not currently in the - // queue. - revert_to_idle!(); - return Ready(Ok(())); - } else { - // The node is pushed into the queue, the final step is - // to set the node's "next" pointer to return the wait - // queue into a consistent state. - - let prev_waiter = - curr.waiter().unwrap_or_else(|| NonNull::from(&*self.stub)); - - let waiter = maybe_waiter.unwrap(); - - // Link the nodes. - // - // Safety: the mpsc algorithm guarantees the old tail of - // the queue is not removed from the queue during the - // push process. - unsafe { - prev_waiter.as_ref().store_next(waiter); - } - - return Pending; - } - } - Err(actual) => { - curr = SemState(actual); - } - } - } + self.permits.load(Ordering::Acquire) } /// Closes the semaphore. This prevents the semaphore from issuing new /// permits and notifies all pending waiters. pub(crate) fn close(&self) { - // Acquire the `rx_lock`, setting the "closed" flag on the lock. - let prev = self.rx_lock.fetch_or(1, AcqRel); + // Acquire the `add_lock`, setting the "closed" flag on the lock. + let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); if prev != 0 { // Another thread has the lock and will be responsible for notifying @@ -342,34 +103,41 @@ impl Semaphore { // TODO: Handle overflow. A panic is not sufficient, the process must // abort. - let prev = self.rx_lock.fetch_add(n << 1, AcqRel); - + let prev = self.add_lock.fetch_add(n << 1, Ordering::AcqRel); if prev != 0 { // Another thread has the lock and will be responsible for notifying // pending waiters. return; } - self.add_permits_locked(n, false); } fn add_permits_locked(&self, mut rem: usize, mut closed: bool) { while rem > 0 || closed { - if closed { - SemState::fetch_set_closed(&self.state, AcqRel); - } // Release the permits and notify - self.add_permits_locked2(rem, closed); + let mut waiters = self.waiters.lock().unwrap(); + while rem > 0 { + let pop = match waiters.last() { + Some(last) => last.assign_permits(&mut rem, closed), + None => { + self.permits.fetch_add(rem, Ordering::Release); + break; + } + }; + if pop { + waiters.pop_back().unwrap(); + } + } let n = rem << 1; let actual = if closed { - let actual = self.rx_lock.fetch_sub(n | 1, AcqRel); + let actual = self.add_lock.fetch_sub(n | 1, Ordering::AcqRel); closed = false; actual } else { - let actual = self.rx_lock.fetch_sub(n, AcqRel); + let actual = self.add_lock.fetch_sub(n, Ordering::AcqRel); closed = actual & 1 == 1; actual }; @@ -378,224 +146,47 @@ impl Semaphore { } } - /// Releases a specific amount of permits to the semaphore - /// - /// This function is called by `add_permits` after the add lock has been - /// acquired. - fn add_permits_locked2(&self, mut n: usize, closed: bool) { - // If closing the semaphore, we want to drain the entire queue. The - // number of permits being assigned doesn't matter. - if closed { - n = usize::MAX; - } - - 'outer: while n > 0 { - unsafe { - let mut head = self.head.with(|head| *head); - let mut next_ptr = head.as_ref().next.load(Acquire); - - let stub = self.stub(); - - if head == stub { - // The stub node indicates an empty queue. Any remaining - // permits get assigned back to the semaphore. - let next = match NonNull::new(next_ptr) { - Some(next) => next, - None => { - // This loop is not part of the standard intrusive mpsc - // channel algorithm. This is where we atomically pop - // the last task and add `n` to the remaining capacity. - // - // This modification to the pop algorithm works because, - // at this point, we have not done any work (only done - // reading). We have a *pretty* good idea that there is - // no concurrent pusher. - // - // The capacity is then atomically added by doing an - // AcqRel CAS on `state`. The `state` cell is the - // linchpin of the algorithm. - // - // By successfully CASing `head` w/ AcqRel, we ensure - // that, if any thread was racing and entered a push, we - // see that and abort pop, retrying as it is - // "inconsistent". - let mut curr = SemState::load(&self.state, Acquire); - - loop { - if curr.has_waiter(&self.stub) { - // A waiter is being added concurrently. - // This is the MPSC queue's "inconsistent" - // state and we must loop and try again. - thread::yield_now(); - continue 'outer; - } - - // If closing, nothing more to do. - if closed { - debug_assert!(curr.is_closed(), "state = {:?}", curr); - return; - } - - let mut next = curr; - next.release_permits(n, &self.stub); - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => return, - Err(actual) => { - curr = SemState(actual); - } - } - } - } - }; - - self.head.with_mut(|head| *head = next); - head = next; - next_ptr = next.as_ref().next.load(Acquire); - } - - // `head` points to a waiter assign permits to the waiter. If - // all requested permits are satisfied, then we can continue, - // otherwise the node stays in the wait queue. - if !head.as_ref().assign_permits(&mut n, closed) { - assert_eq!(n, 0); - return; - } - - if let Some(next) = NonNull::new(next_ptr) { - self.head.with_mut(|head| *head = next); - - self.remove_queued(head, closed); - continue 'outer; - } - - let state = SemState::load(&self.state, Acquire); - - // This must always be a pointer as the wait list is not empty. - let tail = state.waiter().unwrap(); - - if tail != head { - // Inconsistent - thread::yield_now(); - continue 'outer; - } - - self.push_stub(closed); - - next_ptr = head.as_ref().next.load(Acquire); - - if let Some(next) = NonNull::new(next_ptr) { - self.head.with_mut(|head| *head = next); - - self.remove_queued(head, closed); - continue 'outer; - } - - // Inconsistent state, loop - thread::yield_now(); - } - } - } - - /// The wait node has had all of its permits assigned and has been removed - /// from the wait queue. - /// - /// Attempt to remove the QUEUED bit from the node. If additional permits - /// are concurrently requested, the node must be pushed back into the wait - /// queued. - fn remove_queued(&self, waiter: NonNull, closed: bool) { - let mut curr = WaiterState(unsafe { waiter.as_ref() }.state.load(Acquire)); - - loop { - if curr.is_dropped() { - // The Permit dropped, it is on us to release the memory - let _ = unsafe { Box::from_raw(waiter.as_ptr()) }; - return; - } - - // The node is removed from the queue. We attempt to unset the - // queued bit, but concurrently the waiter has requested more - // permits. When the waiter requested more permits, it saw the - // queued bit set so took no further action. This requires us to - // push the node back into the queue. - if curr.permits_to_acquire() > 0 { - // More permits are requested. The waiter must be re-queued - unsafe { - self.push_waiter(waiter, closed); - } - return; + fn poll_acquire( + &self, + cx: &mut Context<'_>, + needed: u16, + node: Pin<&mut Waiter>, + ) -> Poll> { + let mut curr = self.permits.load(Ordering::Acquire); + let remaining = loop { + if curr == CLOSED { + return Ready(Err(AcquireError(()))); } - - let mut next = curr; - next.unset_queued(); - - let w = unsafe { waiter.as_ref() }; - - match w.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => return, - Err(actual) => { - curr = WaiterState(actual); - } + let next = curr.saturating_sub(needed as usize); + match self.permits.compare_exchange_weak( + curr, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) if curr >= needed as usize => return Ready(Ok(())), + Ok(_) => break needed - curr as u16, + Err(actual) => curr = actual, } + }; + // if we've not returned already, then we need to push the waiter to the + // wait queue. + assert!(node.is_unlinked()); + { + let mut waiter = unsafe { &*node.get() }; + waiter.state.compare_exchange(0, remaining, Ordering::Release).expect("unlinked node should not want permits!"); + waiter.waker.register_by_ref(cx.waker()); } - } - - unsafe fn push_stub(&self, closed: bool) { - self.push_waiter(self.stub(), closed); - } - unsafe fn push_waiter(&self, waiter: NonNull, closed: bool) { - // Set the next pointer. This does not require an atomic operation as - // this node is not accessible. The write will be flushed with the next - // operation - waiter.as_ref().next.store(ptr::null_mut(), Relaxed); - - // Update the tail to point to the new node. We need to see the previous - // node in order to update the next pointer as well as release `task` - // to any other threads calling `push`. - let next = SemState::new_ptr(waiter, closed); - let prev = SemState(self.state.swap(next.0, AcqRel)); - - debug_assert_eq!(closed, prev.is_closed()); - - // This function is only called when there are pending tasks. Because of - // this, the state must *always* be in pointer mode. - let prev = prev.waiter().unwrap(); - - // No cycles plz - debug_assert_ne!(prev, waiter); - - // Release `task` to the consume end. - prev.as_ref().next.store(waiter.as_ptr(), Release); - } - - fn stub(&self) -> NonNull { - unsafe { NonNull::new_unchecked(&*self.stub as *const _ as *mut _) } - } -} + let mut queue = self.waiters.lock().unwrap(); + unsafe { + queue.push_front(node); + } -impl Drop for Semaphore { - fn drop(&mut self) { - self.close(); + Pending } } -impl fmt::Debug for Semaphore { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("Semaphore") - .field("state", &SemState::load(&self.state, Relaxed)) - .field("head", &self.head.with(|ptr| ptr)) - .field("rx_lock", &self.rx_lock.load(Relaxed)) - .field("stub", &self.stub) - .finish() - } -} - -unsafe impl Send for Semaphore {} -unsafe impl Sync for Semaphore {} - -// ===== impl Permit ===== - impl Permit { /// Creates a new `Permit`. /// @@ -604,7 +195,10 @@ impl Permit { use PermitState::Acquired; Permit { - waiter: None, + node: linked_list::Entry::new(Waiter { + waker: AtomicWaker::new(), + needed: 0, + }), state: Acquired(0), } } @@ -617,80 +211,16 @@ impl Permit { } } + /// Tries to acquire the permit. If no permits are available, the current task /// is notified once a new permit becomes available. pub(crate) fn poll_acquire( - &mut self, + self: Pin<&mut Self>, cx: &mut Context<'_>, num_permits: u16, semaphore: &Semaphore, ) -> Poll> { - use std::cmp::Ordering::*; - use PermitState::*; - - match self.state { - Waiting(requested) => { - // There must be a waiter - let waiter = self.waiter.as_ref().unwrap(); - - match requested.cmp(&num_permits) { - Less => { - let delta = num_permits - requested; - - // Request additional permits. If the waiter has been - // dequeued, it must be re-queued. - if !waiter.try_inc_permits_to_acquire(delta as usize) { - let waiter = NonNull::from(&**waiter); - - // Ignore the result. The check for - // `permits_to_acquire()` will converge the state as - // needed - let _ = semaphore.poll_acquire2(delta, || Some(waiter))?; - } - - self.state = Waiting(num_permits); - } - Greater => { - let delta = requested - num_permits; - let to_release = waiter.try_dec_permits_to_acquire(delta as usize); - - semaphore.add_permits(to_release); - self.state = Waiting(num_permits); - } - Equal => {} - } - - if waiter.permits_to_acquire()? == 0 { - self.state = Acquired(requested); - return Ready(Ok(())); - } - - waiter.waker.register_by_ref(cx.waker()); - - if waiter.permits_to_acquire()? == 0 { - self.state = Acquired(requested); - return Ready(Ok(())); - } - - Pending - } - Acquired(acquired) => { - if acquired >= num_permits { - Ready(Ok(())) - } else { - match semaphore.poll_acquire(cx, num_permits - acquired, self)? { - Ready(()) => { - self.state = Acquired(num_permits); - Ready(Ok(())) - } - Pending => { - self.state = Waiting(num_permits); - Pending - } - } - } - } - } + semaphore.poll_acquire(cx, num_permits, self.project().node) } /// Tries to acquire the permit. @@ -699,49 +229,12 @@ impl Permit { num_permits: u16, semaphore: &Semaphore, ) -> Result<(), TryAcquireError> { - use PermitState::*; - - match self.state { - Waiting(requested) => { - // There must be a waiter - let waiter = self.waiter.as_ref().unwrap(); - - if requested > num_permits { - let delta = requested - num_permits; - let to_release = waiter.try_dec_permits_to_acquire(delta as usize); - - semaphore.add_permits(to_release); - self.state = Waiting(num_permits); - } - - let res = waiter.permits_to_acquire().map_err(to_try_acquire)?; - - if res == 0 { - if requested < num_permits { - // Try to acquire the additional permits - semaphore.try_acquire(num_permits - requested)?; - } - - self.state = Acquired(num_permits); - Ok(()) - } else { - Err(TryAcquireError::NoPermits) - } - } - Acquired(acquired) => { - if acquired < num_permits { - semaphore.try_acquire(num_permits - acquired)?; - self.state = Acquired(num_permits); - } - - Ok(()) - } - } + unimplemented!() } /// Releases a permit back to the semaphore pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { - let n = self.forget(n); + let n = self.forget(n, semaphore); semaphore.add_permits(n as usize); } @@ -755,7 +248,7 @@ impl Permit { /// /// Will forget **at most** the number of acquired permits. This number is /// returned. - pub(crate) fn forget(&mut self, n: u16) -> u16 { + pub(crate) fn forget(&mut self, n: u16, semaphore: &Semaphore) -> u16 { use PermitState::*; match self.state { @@ -764,13 +257,13 @@ impl Permit { // Decrement let acquired = self - .waiter - .as_ref() - .unwrap() + .node .try_dec_permits_to_acquire(n as usize) as u16; if n == requested { self.state = Acquired(0); + // TODO: rm from wait list here! + semaphore } else if acquired == requested - n { self.state = Waiting(acquired); } else { @@ -794,15 +287,55 @@ impl Default for Permit { } } -impl Drop for Permit { - fn drop(&mut self) { - if let Some(waiter) = self.waiter.take() { - // Set the dropped flag - let state = WaiterState(waiter.state.fetch_or(DROPPED, AcqRel)); +impl Waiter { + /// Assign permits to the waiter. + /// + /// Returns `true` if the waiter should be removed from the queue + fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { + let mut curr = self.state.load(Ordering::Acquire); + + loop { + // Number of permits to assign to this waiter + let assign = cmp::min(curr, *n); + let next = curr - assign; + match self.state.compare_exchange(curr, next, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => { + // Update `n` + *n -= assign; - if state.is_queued() { - // The waiter is stored in the queue. The semaphore will drop it - std::mem::forget(waiter); + if next == 0 { + if curr > 0 { + self.waker.wake(); + } + + return true; + } else { + return false; + } + } + Err(actual) => curr = actual, + } + } + } + + /// Try to decrement the number of permits to acquire. This returns the + /// actual number of permits that were decremented. The delta betweeen `n` + /// and the return has been assigned to the permit and the caller must + /// assign these back to the semaphore. + fn try_dec_permits_to_acquire(&self, n: usize) -> usize { + let mut curr = self.state.load(Ordering::Acquire); + + loop { + if !curr.is_queued() { + assert_eq!(0, curr.permits_to_acquire()); + } + + let delta = cmp::min(n, curr.permits_to_acquire()); + let rem = curr.permits_to_acquire() - delta; + + match self.state.compare_exchange(curr, rem, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return n - delta, + Err(actual) => curr = actual, } } } @@ -859,361 +392,3 @@ impl fmt::Display for TryAcquireError { } impl std::error::Error for TryAcquireError {} - -// ===== impl Waiter ===== - -impl Waiter { - fn new() -> Waiter { - Waiter { - state: AtomicUsize::new(0), - waker: AtomicWaker::new(), - next: AtomicPtr::new(ptr::null_mut()), - } - } - - fn permits_to_acquire(&self) -> Result { - let state = WaiterState(self.state.load(Acquire)); - - if state.is_closed() { - Err(AcquireError(())) - } else { - Ok(state.permits_to_acquire()) - } - } - - /// Only increments the number of permits *if* the waiter is currently - /// queued. - /// - /// # Returns - /// - /// `true` if the number of permits to acquire has been incremented. `false` - /// otherwise. On `false`, the caller should use `Semaphore::poll_acquire`. - fn try_inc_permits_to_acquire(&self, n: usize) -> bool { - let mut curr = WaiterState(self.state.load(Acquire)); - - loop { - if !curr.is_queued() { - assert_eq!(0, curr.permits_to_acquire()); - return false; - } - - let mut next = curr; - next.set_permits_to_acquire(n + curr.permits_to_acquire()); - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => return true, - Err(actual) => curr = WaiterState(actual), - } - } - } - - /// Try to decrement the number of permits to acquire. This returns the - /// actual number of permits that were decremented. The delta betweeen `n` - /// and the return has been assigned to the permit and the caller must - /// assign these back to the semaphore. - fn try_dec_permits_to_acquire(&self, n: usize) -> usize { - let mut curr = WaiterState(self.state.load(Acquire)); - - loop { - if !curr.is_queued() { - assert_eq!(0, curr.permits_to_acquire()); - } - - let delta = cmp::min(n, curr.permits_to_acquire()); - let rem = curr.permits_to_acquire() - delta; - - let mut next = curr; - next.set_permits_to_acquire(rem); - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => return n - delta, - Err(actual) => curr = WaiterState(actual), - } - } - } - - /// Store the number of remaining permits needed to satisfy the waiter and - /// transition to the "QUEUED" state. - /// - /// # Returns - /// - /// `true` if the `QUEUED` bit was set as part of the transition. - fn to_queued(&self, num_permits: usize) -> bool { - let mut curr = WaiterState(self.state.load(Acquire)); - - // The waiter should **not** be waiting for any permits. - debug_assert_eq!(curr.permits_to_acquire(), 0); - - loop { - let mut next = curr; - next.set_permits_to_acquire(num_permits); - next.set_queued(); - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => { - if curr.is_queued() { - return false; - } else { - // Make sure the next pointer is null - self.next.store(ptr::null_mut(), Relaxed); - return true; - } - } - Err(actual) => curr = WaiterState(actual), - } - } - } - - /// Set the number of permits to acquire. - /// - /// This function is only called when the waiter is being inserted into the - /// wait queue. Because of this, there are no concurrent threads that can - /// modify the state and using `store` is safe. - fn set_permits_to_acquire(&self, num_permits: usize) { - debug_assert!(WaiterState(self.state.load(Acquire)).is_queued()); - - let mut state = WaiterState(QUEUED); - state.set_permits_to_acquire(num_permits); - - self.state.store(state.0, Release); - } - - /// Assign permits to the waiter. - /// - /// Returns `true` if the waiter should be removed from the queue - fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { - let mut curr = WaiterState(self.state.load(Acquire)); - - loop { - let mut next = curr; - - // Number of permits to assign to this waiter - let assign = cmp::min(curr.permits_to_acquire(), *n); - - // Assign the permits - next.set_permits_to_acquire(curr.permits_to_acquire() - assign); - - if closed { - next.set_closed(); - } - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => { - // Update `n` - *n -= assign; - - if next.permits_to_acquire() == 0 { - if curr.permits_to_acquire() > 0 { - self.waker.wake(); - } - - return true; - } else { - return false; - } - } - Err(actual) => curr = WaiterState(actual), - } - } - } - - fn revert_to_idle(&self) { - // An idle node is not waiting on any permits - self.state.store(0, Relaxed); - } - - fn store_next(&self, next: NonNull) { - self.next.store(next.as_ptr(), Release); - } -} - -// ===== impl SemState ===== - -impl SemState { - /// Returns a new default `State` value. - fn new(permits: usize, stub: &Waiter) -> SemState { - assert!(permits <= MAX_PERMITS); - - if permits > 0 { - SemState((permits << NUM_SHIFT) | NUM_FLAG) - } else { - SemState(stub as *const _ as usize) - } - } - - /// Returns a `State` tracking `ptr` as the tail of the queue. - fn new_ptr(tail: NonNull, closed: bool) -> SemState { - let mut val = tail.as_ptr() as usize; - - if closed { - val |= CLOSED_FLAG; - } - - SemState(val) - } - - /// Returns the amount of remaining capacity - fn available_permits(self) -> usize { - if !self.has_available_permits() { - return 0; - } - - self.0 >> NUM_SHIFT - } - - /// Returns `true` if the state has permits that can be claimed by a waiter. - fn has_available_permits(self) -> bool { - self.0 & NUM_FLAG == NUM_FLAG - } - - fn has_waiter(self, stub: &Waiter) -> bool { - !self.has_available_permits() && !self.is_stub(stub) - } - - /// Tries to atomically acquire specified number of permits. - /// - /// # Return - /// - /// Returns `true` if the specified number of permits were acquired, `false` - /// otherwise. Returning false does not mean that there are no more - /// available permits. - fn acquire_permits(&mut self, num: usize, stub: &Waiter) -> bool { - debug_assert!(num > 0); - - if self.available_permits() < num { - return false; - } - - debug_assert!(self.waiter().is_none()); - - self.0 -= num << NUM_SHIFT; - - if self.0 == NUM_FLAG { - // Set the state to the stub pointer. - self.0 = stub as *const _ as usize; - } - - true - } - - /// Releases permits - /// - /// Returns `true` if the permits were accepted. - fn release_permits(&mut self, permits: usize, stub: &Waiter) { - debug_assert!(permits > 0); - - if self.is_stub(stub) { - self.0 = (permits << NUM_SHIFT) | NUM_FLAG | (self.0 & CLOSED_FLAG); - return; - } - - debug_assert!(self.has_available_permits()); - - self.0 += permits << NUM_SHIFT; - } - - fn is_waiter(self) -> bool { - self.0 & NUM_FLAG == 0 - } - - /// Returns the waiter, if one is set. - fn waiter(self) -> Option> { - if self.is_waiter() { - let waiter = NonNull::new(self.as_ptr()).expect("null pointer stored"); - - Some(waiter) - } else { - None - } - } - - /// Assumes `self` represents a pointer - fn as_ptr(self) -> *mut Waiter { - (self.0 & !CLOSED_FLAG) as *mut Waiter - } - - /// Sets to a pointer to a waiter. - /// - /// This can only be done from the full state. - fn set_waiter(&mut self, waiter: NonNull) { - let waiter = waiter.as_ptr() as usize; - debug_assert!(!self.is_closed()); - - self.0 = waiter; - } - - fn is_stub(self, stub: &Waiter) -> bool { - self.as_ptr() as usize == stub as *const _ as usize - } - - /// Loads the state from an AtomicUsize. - fn load(cell: &AtomicUsize, ordering: Ordering) -> SemState { - let value = cell.load(ordering); - SemState(value) - } - - fn fetch_set_closed(cell: &AtomicUsize, ordering: Ordering) -> SemState { - let value = cell.fetch_or(CLOSED_FLAG, ordering); - SemState(value) - } - - fn is_closed(self) -> bool { - self.0 & CLOSED_FLAG == CLOSED_FLAG - } - - /// Converts the state into a `usize` representation. - fn to_usize(self) -> usize { - self.0 - } -} - -impl fmt::Debug for SemState { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut fmt = fmt.debug_struct("SemState"); - - if self.is_waiter() { - fmt.field("state", &""); - } else { - fmt.field("permits", &self.available_permits()); - } - - fmt.finish() - } -} - -// ===== impl WaiterState ===== - -impl WaiterState { - fn permits_to_acquire(self) -> usize { - self.0 >> PERMIT_SHIFT - } - - fn set_permits_to_acquire(&mut self, val: usize) { - self.0 = (val << PERMIT_SHIFT) | (self.0 & !PERMIT_MASK) - } - - fn is_queued(self) -> bool { - self.0 & QUEUED == QUEUED - } - - fn set_queued(&mut self) { - self.0 |= QUEUED; - } - - fn is_closed(self) -> bool { - self.0 & CLOSED == CLOSED - } - - fn set_closed(&mut self) { - self.0 |= CLOSED; - } - - fn unset_queued(&mut self) { - assert!(self.is_queued()); - self.0 -= QUEUED; - } - - fn is_dropped(self) -> bool { - self.0 & DROPPED == DROPPED - } -} diff --git a/tokio/src/sync/semaphore_ll2.rs b/tokio/src/sync/semaphore_ll2.rs deleted file mode 100644 index a2d51462bbd..00000000000 --- a/tokio/src/sync/semaphore_ll2.rs +++ /dev/null @@ -1,330 +0,0 @@ -use crate::loom::{ - alloc, - future::AtomicWaker, - sync::{atomic::AtomicUsize, Mutex}, -}; -use crate::util::linked_list::{self, LinkedList}; - -use std::{ - pin::Pin, - ptr::NonNull, - sync::atomic::Ordering, - task::{ - Context, Poll, - Poll::{Pending, Ready}, - }, -}; - -pub(crate) struct Semaphore { - waiters: Mutex>>, - permits: AtomicUsize, - add_lock: AtomicUsize, -} - -/// Error returned by `Permit::poll_acquire`. -#[derive(Debug)] -pub(crate) struct AcquireError(()); - -pin_project_lite::pin_project! { - pub(crate) struct Permit { - #[pin] - node: WaitNode, - state: PermitState, - } -} - -type WaitNode = linked_list::Entry>; - -/// Permit state -#[derive(Debug, Copy, Clone)] -enum PermitState { - /// Currently waiting for permits to be made available and assigned to the - /// waiter. - Waiting(u16), - - /// The number of acquired permits - Acquired(u16), -} - -struct Waiter { - waker: AtomicWaker, - needed: u16, -} - -const CLOSED: usize = std::usize::MAX; - -impl Semaphore { - /// Creates a new semaphore with the initial number of permits - /// - /// # Panics - /// - /// Panics if `permits` is zero. - pub(crate) fn new(permits: usize) -> Self { - assert!(permits > 0, "number of permits must be greater than 0"); - Self { - permits: AtomicUsize::new(permits), - waiters: Mutex::new(LinkedList::new()), - add_lock: AtomicUsize::new(0), - } - } - - /// Returns the current number of available permits - pub(crate) fn available_permits(&self) -> usize { - self.permits.load(Ordering::Acquire) - } - - /// Closes the semaphore. This prevents the semaphore from issuing new - /// permits and notifies all pending waiters. - pub(crate) fn close(&self) { - // Acquire the `add_lock`, setting the "closed" flag on the lock. - let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); - - if prev != 0 { - // Another thread has the lock and will be responsible for notifying - // pending waiters. - return; - } - - self.add_permits_locked(0, true); - } - - /// Adds `n` new permits to the semaphore. - pub(crate) fn add_permits(&self, n: usize) { - if n == 0 { - return; - } - - // TODO: Handle overflow. A panic is not sufficient, the process must - // abort. - let prev = self.add_lock.fetch_add(n << 1, Ordering::AcqRel); - if prev != 0 { - // Another thread has the lock and will be responsible for notifying - // pending waiters. - return; - } - self.add_permits_locked(n, false); - } - - fn add_permits_locked(&self, mut rem: usize, mut closed: bool) { - while rem > 0 || closed { - if closed { - SemState::fetch_set_closed(&self.state, AcqRel); - } - - // Release the permits and notify - let mut waiters = self.waiters.lock().unwrap(); - while rem > 0 { - let pop = { - let mut last = match waiters.last_mut() { - Some(mut last) => last, - None => { - self.permits.fetch_add(rem, Ordering::Release); - break; - } - }; - if rem >= last.needed { - rem -= last.needed; - last.needed = 0; - true - } else { - last.neede -= rem; - break; - } - }; - if pop { - waiters.pop_back().unwrap().get_mut().waker.wake(); - } - } - - let n = rem << 1; - - let actual = if closed { - let actual = self.add_lock.fetch_sub(n | 1, AcqRel); - closed = false; - actual - } else { - let actual = self.add_lock.fetch_sub(n, AcqRel); - closed = actual & 1 == 1; - actual - }; - - rem = (actual >> 1) - rem; - } - } - - fn poll_acquire( - &self, - cx: &mut Context<'_>, - needed: u16, - node: Pin<&mut WaitNode>, - ) -> Poll> { - let mut curr = self.permits.load(Ordering::Acquire); - let remaining = loop { - if curr == CLOSED { - return Ready(Err(AcquireError(()))); - } - let next = curr.saturating_sub(needed as usize); - match self.permits.compare_exchange_weak( - curr, - next, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) if curr >= needed as usize => return Ready(Ok(())), - Ok(_) => break needed - curr as u16, - Err(actual) => curr = actual, - } - }; - // if we've not returned already, then we need to push the waiter to the - // wait queue. - assert!(node.is_unlinked()); - { - let mut waiter = unsafe { &mut *node.get() }.get_mut(); - waiter.needed = remaining; - waiter.waker.register_by_ref(cx.waker()); - } - - let mut queue = self.waiters.lock().unwrap(); - unsafe { - queue.push_front(node); - } - - Pending - } -} - -impl Permit { - /// Creates a new `Permit`. - /// - /// The permit begins in the "unacquired" state. - pub(crate) fn new() -> Permit { - use PermitState::Acquired; - - Permit { - node: linked_list::Entry::new(alloc::Track::new(Waiter { - waker: AtomicWaker::new(), - needed: 0, - })), - state: Acquired(0), - } - } - - /// Returns `true` if the permit has been acquired - pub(crate) fn is_acquired(&self) -> bool { - match self.state { - PermitState::Acquired(num) if num > 0 => true, - _ => false, - } - } - - /// Tries to acquire the permit. If no permits are available, the current task - /// is notified once a new permit becomes available. - pub(crate) fn poll_acquire( - &mut self, - cx: &mut Context<'_>, - num_permits: u16, - semaphore: &Semaphore, - ) -> Poll> { - use std::cmp::Ordering::*; - use PermitState::*; - - match self.state { - Waiting(requested) => { - // There must be a waiter - let waiter = self.waiter.as_ref().unwrap(); - - match requested.cmp(&num_permits) { - Less => { - let delta = num_permits - requested; - - // Request additional permits. If the waiter has been - // dequeued, it must be re-queued. - if !waiter.try_inc_permits_to_acquire(delta as usize) { - let waiter = NonNull::from(&**waiter); - - // Ignore the result. The check for - // `permits_to_acquire()` will converge the state as - // needed - let _ = semaphore.poll_acquire2(delta, || Some(waiter))?; - } - - self.state = Waiting(num_permits); - } - Greater => { - let delta = requested - num_permits; - let to_release = waiter.try_dec_permits_to_acquire(delta as usize); - - semaphore.add_permits(to_release); - self.state = Waiting(num_permits); - } - Equal => {} - } - - if waiter.permits_to_acquire()? == 0 { - self.state = Acquired(requested); - return Ready(Ok(())); - } - - waiter.waker.register_by_ref(cx.waker()); - - if waiter.permits_to_acquire()? == 0 { - self.state = Acquired(requested); - return Ready(Ok(())); - } - - Pending - } - Acquired(acquired) => { - if acquired >= num_permits { - Ready(Ok(())) - } else { - match semaphore.poll_acquire(cx, num_permits - acquired, self)? { - Ready(()) => { - self.state = Acquired(num_permits); - Ready(Ok(())) - } - Pending => { - self.state = Waiting(num_permits); - Pending - } - } - } - } - } - } - - /// Tries to acquire the permit. - pub(crate) fn try_acquire( - &mut self, - num_permits: u16, - semaphore: &Semaphore, - ) -> Result<(), TryAcquireError> { - unimplemented!() - } - - /// Releases a permit back to the semaphore - pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { - let n = self.forget(n); - semaphore.add_permits(n as usize); - } - - /// Forgets the permit **without** releasing it back to the semaphore. - /// - /// After calling `forget`, `poll_acquire` is able to acquire new permit - /// from the sempahore. - /// - /// Repeatedly calling `forget` without associated calls to `add_permit` - /// will result in the semaphore losing all permits. - /// - /// Will forget **at most** the number of acquired permits. This number is - /// returned. - pub(crate) fn forget(&mut self, n: u16) -> u16 { - unimplemented!() - } -} - -impl Default for Permit { - fn default() -> Self { - Self::new() - } -} diff --git a/tokio/src/sync/semaphore_ll_old.rs b/tokio/src/sync/semaphore_ll_old.rs new file mode 100644 index 00000000000..69fd4a6a5d3 --- /dev/null +++ b/tokio/src/sync/semaphore_ll_old.rs @@ -0,0 +1,1219 @@ +#![cfg_attr(not(feature = "sync"), allow(dead_code, unreachable_pub))] + +//! Thread-safe, asynchronous counting semaphore. +//! +//! A `Semaphore` instance holds a set of permits. Permits are used to +//! synchronize access to a shared resource. +//! +//! Before accessing the shared resource, callers acquire a permit from the +//! semaphore. Once the permit is acquired, the caller then enters the critical +//! section. If no permits are available, then acquiring the semaphore returns +//! `Pending`. The task is woken once a permit becomes available. + +use crate::loom::cell::CausalCell; +use crate::loom::future::AtomicWaker; +use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize}; +use crate::loom::thread; + +use std::cmp; +use std::fmt; +use std::ptr::{self, NonNull}; +use std::sync::atomic::Ordering::{self, AcqRel, Acquire, Relaxed, Release}; +use std::task::Poll::{Pending, Ready}; +use std::task::{Context, Poll}; +use std::usize; + +/// Futures-aware semaphore. +pub(crate) struct Semaphore { + /// Tracks both the waiter queue tail pointer and the number of remaining + /// permits. + state: AtomicUsize, + + /// waiter queue head pointer. + head: CausalCell>, + + /// Coordinates access to the queue head. + rx_lock: AtomicUsize, + + /// Stub waiter node used as part of the MPSC channel algorithm. + stub: Box, +} + +/// A semaphore permit +/// +/// Tracks the lifecycle of a semaphore permit. +/// +/// An instance of `Permit` is intended to be used with a **single** instance of +/// `Semaphore`. Using a single instance of `Permit` with multiple semaphore +/// instances will result in unexpected behavior. +/// +/// `Permit` does **not** release the permit back to the semaphore on drop. It +/// is the user's responsibility to ensure that `Permit::release` is called +/// before dropping the permit. +#[derive(Debug)] +pub(crate) struct Permit { + waiter: Option>, + state: PermitState, +} + +/// Error returned by `Permit::poll_acquire`. +#[derive(Debug)] +pub(crate) struct AcquireError(()); + +/// Error returned by `Permit::try_acquire`. +#[derive(Debug)] +pub(crate) enum TryAcquireError { + Closed, + NoPermits, +} + +/// Node used to notify the semaphore waiter when permit is available. +#[derive(Debug)] +struct Waiter { + /// Stores waiter state. + /// + /// See `WaiterState` for more details. + state: AtomicUsize, + + /// Task to wake when a permit is made available. + waker: AtomicWaker, + + /// Next pointer in the queue of waiting senders. + next: AtomicPtr, +} + +/// Semaphore state +/// +/// The 2 low bits track the modes. +/// +/// - Closed +/// - Full +/// +/// When not full, the rest of the `usize` tracks the total number of messages +/// in the channel. When full, the rest of the `usize` is a pointer to the tail +/// of the "waiting senders" queue. +#[derive(Copy, Clone)] +struct SemState(usize); + +/// Permit state +#[derive(Debug, Copy, Clone)] +enum PermitState { + /// Currently waiting for permits to be made available and assigned to the + /// waiter. + Waiting(u16), + + /// The number of acquired permits + Acquired(u16), +} + +/// State for an individual waker node +#[derive(Debug, Copy, Clone)] +struct WaiterState(usize); + +/// Waiter node is in the semaphore queue +const QUEUED: usize = 0b001; + +/// Semaphore has been closed, no more permits will be issued. +const CLOSED: usize = 0b10; + +/// The permit that owns the `Waiter` dropped. +const DROPPED: usize = 0b100; + +/// Represents "one requested permit" in the waiter state +const PERMIT_ONE: usize = 0b1000; + +/// Masks the waiter state to only contain bits tracking number of requested +/// permits. +const PERMIT_MASK: usize = usize::MAX - (PERMIT_ONE - 1); + +/// How much to shift a permit count to pack it into the waker state +const PERMIT_SHIFT: u32 = PERMIT_ONE.trailing_zeros(); + +/// Flag differentiating between available permits and waiter pointers. +/// +/// If we assume pointers are properly aligned, then the least significant bit +/// will always be zero. So, we use that bit to track if the value represents a +/// number. +const NUM_FLAG: usize = 0b01; + +/// Signal the semaphore is closed +const CLOSED_FLAG: usize = 0b10; + +/// Maximum number of permits a semaphore can manage +const MAX_PERMITS: usize = usize::MAX >> NUM_SHIFT; + +/// When representing "numbers", the state has to be shifted this much (to get +/// rid of the flag bit). +const NUM_SHIFT: usize = 2; + +// ===== impl Semaphore ===== + +impl Semaphore { + /// Creates a new semaphore with the initial number of permits + /// + /// # Panics + /// + /// Panics if `permits` is zero. + pub(crate) fn new(permits: usize) -> Semaphore { + let stub = Box::new(Waiter::new()); + let ptr = NonNull::from(&*stub); + + // Allocations are aligned + debug_assert!(ptr.as_ptr() as usize & NUM_FLAG == 0); + + let state = SemState::new(permits, &stub); + + Semaphore { + state: AtomicUsize::new(state.to_usize()), + head: CausalCell::new(ptr), + rx_lock: AtomicUsize::new(0), + stub, + } + } + + /// Returns the current number of available permits + pub(crate) fn available_permits(&self) -> usize { + let curr = SemState(self.state.load(Acquire)); + curr.available_permits() + } + + /// Tries to acquire the requested number of permits, registering the waiter + /// if not enough permits are available. + fn poll_acquire( + &self, + cx: &mut Context<'_>, + num_permits: u16, + permit: &mut Permit, + ) -> Poll> { + self.poll_acquire2(num_permits, || { + let waiter = permit.waiter.get_or_insert_with(|| Box::new(Waiter::new())); + + waiter.waker.register_by_ref(cx.waker()); + + Some(NonNull::from(&**waiter)) + }) + } + + fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { + match self.poll_acquire2(num_permits, || None) { + Poll::Ready(res) => res.map_err(to_try_acquire), + Poll::Pending => Err(TryAcquireError::NoPermits), + } + } + + /// Polls for a permit + /// + /// Tries to acquire available permits first. If unable to acquire a + /// sufficient number of permits, the caller's waiter is pushed onto the + /// semaphore's wait queue. + fn poll_acquire2( + &self, + num_permits: u16, + mut get_waiter: F, + ) -> Poll> + where + F: FnMut() -> Option>, + { + let num_permits = num_permits as usize; + + // Load the current state + let mut curr = SemState(self.state.load(Acquire)); + + // Saves a ref to the waiter node + let mut maybe_waiter: Option> = None; + + /// Used in branches where we attempt to push the waiter into the wait + /// queue but fail due to permits becoming available or the wait queue + /// transitioning to "closed". In this case, the waiter must be + /// transitioned back to the "idle" state. + macro_rules! revert_to_idle { + () => { + if let Some(waiter) = maybe_waiter { + unsafe { waiter.as_ref() }.revert_to_idle(); + } + }; + } + + loop { + let mut next = curr; + + if curr.is_closed() { + revert_to_idle!(); + return Ready(Err(AcquireError::closed())); + } + + let acquired = next.acquire_permits(num_permits, &self.stub); + + if !acquired { + // There are not enough available permits to satisfy the + // request. The permit transitions to a waiting state. + debug_assert!(curr.waiter().is_some() || curr.available_permits() < num_permits); + + if let Some(waiter) = maybe_waiter.as_ref() { + // Safety: the caller owns the waiter. + let w = unsafe { waiter.as_ref() }; + w.set_permits_to_acquire(num_permits - curr.available_permits()); + } else { + // Get the waiter for the permit. + if let Some(waiter) = get_waiter() { + // Safety: the caller owns the waiter. + let w = unsafe { waiter.as_ref() }; + + // If there are any currently available permits, the + // waiter acquires those immediately and waits for the + // remaining permits to become available. + if !w.to_queued(num_permits - curr.available_permits()) { + // The node is alrady queued, there is no further work + // to do. + return Pending; + } + + maybe_waiter = Some(waiter); + } else { + // No waiter, this indicates the caller does not wish to + // "wait", so there is nothing left to do. + return Pending; + } + } + + next.set_waiter(maybe_waiter.unwrap()); + } + + debug_assert_ne!(curr.0, 0); + debug_assert_ne!(next.0, 0); + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => { + if acquired { + // Successfully acquire permits **without** queuing the + // waiter node. The waiter node is not currently in the + // queue. + revert_to_idle!(); + return Ready(Ok(())); + } else { + // The node is pushed into the queue, the final step is + // to set the node's "next" pointer to return the wait + // queue into a consistent state. + + let prev_waiter = + curr.waiter().unwrap_or_else(|| NonNull::from(&*self.stub)); + + let waiter = maybe_waiter.unwrap(); + + // Link the nodes. + // + // Safety: the mpsc algorithm guarantees the old tail of + // the queue is not removed from the queue during the + // push process. + unsafe { + prev_waiter.as_ref().store_next(waiter); + } + + return Pending; + } + } + Err(actual) => { + curr = SemState(actual); + } + } + } + } + + /// Closes the semaphore. This prevents the semaphore from issuing new + /// permits and notifies all pending waiters. + pub(crate) fn close(&self) { + // Acquire the `rx_lock`, setting the "closed" flag on the lock. + let prev = self.rx_lock.fetch_or(1, AcqRel); + + if prev != 0 { + // Another thread has the lock and will be responsible for notifying + // pending waiters. + return; + } + + self.add_permits_locked(0, true); + } + + /// Adds `n` new permits to the semaphore. + pub(crate) fn add_permits(&self, n: usize) { + if n == 0 { + return; + } + + // TODO: Handle overflow. A panic is not sufficient, the process must + // abort. + let prev = self.rx_lock.fetch_add(n << 1, AcqRel); + + if prev != 0 { + // Another thread has the lock and will be responsible for notifying + // pending waiters. + return; + } + + self.add_permits_locked(n, false); + } + + fn add_permits_locked(&self, mut rem: usize, mut closed: bool) { + while rem > 0 || closed { + if closed { + SemState::fetch_set_closed(&self.state, AcqRel); + } + + // Release the permits and notify + self.add_permits_locked2(rem, closed); + + let n = rem << 1; + + let actual = if closed { + let actual = self.rx_lock.fetch_sub(n | 1, AcqRel); + closed = false; + actual + } else { + let actual = self.rx_lock.fetch_sub(n, AcqRel); + closed = actual & 1 == 1; + actual + }; + + rem = (actual >> 1) - rem; + } + } + + /// Releases a specific amount of permits to the semaphore + /// + /// This function is called by `add_permits` after the add lock has been + /// acquired. + fn add_permits_locked2(&self, mut n: usize, closed: bool) { + // If closing the semaphore, we want to drain the entire queue. The + // number of permits being assigned doesn't matter. + if closed { + n = usize::MAX; + } + + 'outer: while n > 0 { + unsafe { + let mut head = self.head.with(|head| *head); + let mut next_ptr = head.as_ref().next.load(Acquire); + + let stub = self.stub(); + + if head == stub { + // The stub node indicates an empty queue. Any remaining + // permits get assigned back to the semaphore. + let next = match NonNull::new(next_ptr) { + Some(next) => next, + None => { + // This loop is not part of the standard intrusive mpsc + // channel algorithm. This is where we atomically pop + // the last task and add `n` to the remaining capacity. + // + // This modification to the pop algorithm works because, + // at this point, we have not done any work (only done + // reading). We have a *pretty* good idea that there is + // no concurrent pusher. + // + // The capacity is then atomically added by doing an + // AcqRel CAS on `state`. The `state` cell is the + // linchpin of the algorithm. + // + // By successfully CASing `head` w/ AcqRel, we ensure + // that, if any thread was racing and entered a push, we + // see that and abort pop, retrying as it is + // "inconsistent". + let mut curr = SemState::load(&self.state, Acquire); + + loop { + if curr.has_waiter(&self.stub) { + // A waiter is being added concurrently. + // This is the MPSC queue's "inconsistent" + // state and we must loop and try again. + thread::yield_now(); + continue 'outer; + } + + // If closing, nothing more to do. + if closed { + debug_assert!(curr.is_closed(), "state = {:?}", curr); + return; + } + + let mut next = curr; + next.release_permits(n, &self.stub); + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => return, + Err(actual) => { + curr = SemState(actual); + } + } + } + } + }; + + self.head.with_mut(|head| *head = next); + head = next; + next_ptr = next.as_ref().next.load(Acquire); + } + + // `head` points to a waiter assign permits to the waiter. If + // all requested permits are satisfied, then we can continue, + // otherwise the node stays in the wait queue. + if !head.as_ref().assign_permits(&mut n, closed) { + assert_eq!(n, 0); + return; + } + + if let Some(next) = NonNull::new(next_ptr) { + self.head.with_mut(|head| *head = next); + + self.remove_queued(head, closed); + continue 'outer; + } + + let state = SemState::load(&self.state, Acquire); + + // This must always be a pointer as the wait list is not empty. + let tail = state.waiter().unwrap(); + + if tail != head { + // Inconsistent + thread::yield_now(); + continue 'outer; + } + + self.push_stub(closed); + + next_ptr = head.as_ref().next.load(Acquire); + + if let Some(next) = NonNull::new(next_ptr) { + self.head.with_mut(|head| *head = next); + + self.remove_queued(head, closed); + continue 'outer; + } + + // Inconsistent state, loop + thread::yield_now(); + } + } + } + + /// The wait node has had all of its permits assigned and has been removed + /// from the wait queue. + /// + /// Attempt to remove the QUEUED bit from the node. If additional permits + /// are concurrently requested, the node must be pushed back into the wait + /// queued. + fn remove_queued(&self, waiter: NonNull, closed: bool) { + let mut curr = WaiterState(unsafe { waiter.as_ref() }.state.load(Acquire)); + + loop { + if curr.is_dropped() { + // The Permit dropped, it is on us to release the memory + let _ = unsafe { Box::from_raw(waiter.as_ptr()) }; + return; + } + + // The node is removed from the queue. We attempt to unset the + // queued bit, but concurrently the waiter has requested more + // permits. When the waiter requested more permits, it saw the + // queued bit set so took no further action. This requires us to + // push the node back into the queue. + if curr.permits_to_acquire() > 0 { + // More permits are requested. The waiter must be re-queued + unsafe { + self.push_waiter(waiter, closed); + } + return; + } + + let mut next = curr; + next.unset_queued(); + + let w = unsafe { waiter.as_ref() }; + + match w.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => return, + Err(actual) => { + curr = WaiterState(actual); + } + } + } + } + + unsafe fn push_stub(&self, closed: bool) { + self.push_waiter(self.stub(), closed); + } + + unsafe fn push_waiter(&self, waiter: NonNull, closed: bool) { + // Set the next pointer. This does not require an atomic operation as + // this node is not accessible. The write will be flushed with the next + // operation + waiter.as_ref().next.store(ptr::null_mut(), Relaxed); + + // Update the tail to point to the new node. We need to see the previous + // node in order to update the next pointer as well as release `task` + // to any other threads calling `push`. + let next = SemState::new_ptr(waiter, closed); + let prev = SemState(self.state.swap(next.0, AcqRel)); + + debug_assert_eq!(closed, prev.is_closed()); + + // This function is only called when there are pending tasks. Because of + // this, the state must *always* be in pointer mode. + let prev = prev.waiter().unwrap(); + + // No cycles plz + debug_assert_ne!(prev, waiter); + + // Release `task` to the consume end. + prev.as_ref().next.store(waiter.as_ptr(), Release); + } + + fn stub(&self) -> NonNull { + unsafe { NonNull::new_unchecked(&*self.stub as *const _ as *mut _) } + } +} + +impl Drop for Semaphore { + fn drop(&mut self) { + self.close(); + } +} + +impl fmt::Debug for Semaphore { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("Semaphore") + .field("state", &SemState::load(&self.state, Relaxed)) + .field("head", &self.head.with(|ptr| ptr)) + .field("rx_lock", &self.rx_lock.load(Relaxed)) + .field("stub", &self.stub) + .finish() + } +} + +unsafe impl Send for Semaphore {} +unsafe impl Sync for Semaphore {} + +// ===== impl Permit ===== + +impl Permit { + /// Creates a new `Permit`. + /// + /// The permit begins in the "unacquired" state. + pub(crate) fn new() -> Permit { + use PermitState::Acquired; + + Permit { + waiter: None, + state: Acquired(0), + } + } + + /// Returns `true` if the permit has been acquired + pub(crate) fn is_acquired(&self) -> bool { + match self.state { + PermitState::Acquired(num) if num > 0 => true, + _ => false, + } + } + + /// Tries to acquire the permit. If no permits are available, the current task + /// is notified once a new permit becomes available. + pub(crate) fn poll_acquire( + &mut self, + cx: &mut Context<'_>, + num_permits: u16, + semaphore: &Semaphore, + ) -> Poll> { + use std::cmp::Ordering::*; + use PermitState::*; + + match self.state { + Waiting(requested) => { + // There must be a waiter + let waiter = self.waiter.as_ref().unwrap(); + + match requested.cmp(&num_permits) { + Less => { + let delta = num_permits - requested; + + // Request additional permits. If the waiter has been + // dequeued, it must be re-queued. + if !waiter.try_inc_permits_to_acquire(delta as usize) { + let waiter = NonNull::from(&**waiter); + + // Ignore the result. The check for + // `permits_to_acquire()` will converge the state as + // needed + let _ = semaphore.poll_acquire2(delta, || Some(waiter))?; + } + + self.state = Waiting(num_permits); + } + Greater => { + let delta = requested - num_permits; + let to_release = waiter.try_dec_permits_to_acquire(delta as usize); + + semaphore.add_permits(to_release); + self.state = Waiting(num_permits); + } + Equal => {} + } + + if waiter.permits_to_acquire()? == 0 { + self.state = Acquired(requested); + return Ready(Ok(())); + } + + waiter.waker.register_by_ref(cx.waker()); + + if waiter.permits_to_acquire()? == 0 { + self.state = Acquired(requested); + return Ready(Ok(())); + } + + Pending + } + Acquired(acquired) => { + if acquired >= num_permits { + Ready(Ok(())) + } else { + match semaphore.poll_acquire(cx, num_permits - acquired, self)? { + Ready(()) => { + self.state = Acquired(num_permits); + Ready(Ok(())) + } + Pending => { + self.state = Waiting(num_permits); + Pending + } + } + } + } + } + } + + /// Tries to acquire the permit. + pub(crate) fn try_acquire( + &mut self, + num_permits: u16, + semaphore: &Semaphore, + ) -> Result<(), TryAcquireError> { + use PermitState::*; + + match self.state { + Waiting(requested) => { + // There must be a waiter + let waiter = self.waiter.as_ref().unwrap(); + + if requested > num_permits { + let delta = requested - num_permits; + let to_release = waiter.try_dec_permits_to_acquire(delta as usize); + + semaphore.add_permits(to_release); + self.state = Waiting(num_permits); + } + + let res = waiter.permits_to_acquire().map_err(to_try_acquire)?; + + if res == 0 { + if requested < num_permits { + // Try to acquire the additional permits + semaphore.try_acquire(num_permits - requested)?; + } + + self.state = Acquired(num_permits); + Ok(()) + } else { + Err(TryAcquireError::NoPermits) + } + } + Acquired(acquired) => { + if acquired < num_permits { + semaphore.try_acquire(num_permits - acquired)?; + self.state = Acquired(num_permits); + } + + Ok(()) + } + } + } + + /// Releases a permit back to the semaphore + pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { + let n = self.forget(n); + semaphore.add_permits(n as usize); + } + + /// Forgets the permit **without** releasing it back to the semaphore. + /// + /// After calling `forget`, `poll_acquire` is able to acquire new permit + /// from the sempahore. + /// + /// Repeatedly calling `forget` without associated calls to `add_permit` + /// will result in the semaphore losing all permits. + /// + /// Will forget **at most** the number of acquired permits. This number is + /// returned. + pub(crate) fn forget(&mut self, n: u16) -> u16 { + use PermitState::*; + + match self.state { + Waiting(requested) => { + let n = cmp::min(n, requested); + + // Decrement + let acquired = self + .waiter + .as_ref() + .unwrap() + .try_dec_permits_to_acquire(n as usize) as u16; + + if n == requested { + self.state = Acquired(0); + } else if acquired == requested - n { + self.state = Waiting(acquired); + } else { + self.state = Waiting(requested - n); + } + + acquired + } + Acquired(acquired) => { + let n = cmp::min(n, acquired); + self.state = Acquired(acquired - n); + n + } + } + } +} + +impl Default for Permit { + fn default() -> Self { + Self::new() + } +} + +impl Drop for Permit { + fn drop(&mut self) { + if let Some(waiter) = self.waiter.take() { + // Set the dropped flag + let state = WaiterState(waiter.state.fetch_or(DROPPED, AcqRel)); + + if state.is_queued() { + // The waiter is stored in the queue. The semaphore will drop it + std::mem::forget(waiter); + } + } + } +} + +// ===== impl AcquireError ==== + +impl AcquireError { + fn closed() -> AcquireError { + AcquireError(()) + } +} + +fn to_try_acquire(_: AcquireError) -> TryAcquireError { + TryAcquireError::Closed +} + +impl fmt::Display for AcquireError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(fmt, "semaphore closed") + } +} + +impl std::error::Error for AcquireError {} + +// ===== impl TryAcquireError ===== + +impl TryAcquireError { + /// Returns `true` if the error was caused by a closed semaphore. + pub(crate) fn is_closed(&self) -> bool { + match self { + TryAcquireError::Closed => true, + _ => false, + } + } + + /// Returns `true` if the error was caused by calling `try_acquire` on a + /// semaphore with no available permits. + pub(crate) fn is_no_permits(&self) -> bool { + match self { + TryAcquireError::NoPermits => true, + _ => false, + } + } +} + +impl fmt::Display for TryAcquireError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TryAcquireError::Closed => write!(fmt, "{}", "semaphore closed"), + TryAcquireError::NoPermits => write!(fmt, "{}", "no permits available"), + } + } +} + +impl std::error::Error for TryAcquireError {} + +// ===== impl Waiter ===== + +impl Waiter { + fn new() -> Waiter { + Waiter { + state: AtomicUsize::new(0), + waker: AtomicWaker::new(), + next: AtomicPtr::new(ptr::null_mut()), + } + } + + fn permits_to_acquire(&self) -> Result { + let state = WaiterState(self.state.load(Acquire)); + + if state.is_closed() { + Err(AcquireError(())) + } else { + Ok(state.permits_to_acquire()) + } + } + + /// Only increments the number of permits *if* the waiter is currently + /// queued. + /// + /// # Returns + /// + /// `true` if the number of permits to acquire has been incremented. `false` + /// otherwise. On `false`, the caller should use `Semaphore::poll_acquire`. + fn try_inc_permits_to_acquire(&self, n: usize) -> bool { + let mut curr = WaiterState(self.state.load(Acquire)); + + loop { + if !curr.is_queued() { + assert_eq!(0, curr.permits_to_acquire()); + return false; + } + + let mut next = curr; + next.set_permits_to_acquire(n + curr.permits_to_acquire()); + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => return true, + Err(actual) => curr = WaiterState(actual), + } + } + } + + /// Try to decrement the number of permits to acquire. This returns the + /// actual number of permits that were decremented. The delta betweeen `n` + /// and the return has been assigned to the permit and the caller must + /// assign these back to the semaphore. + fn try_dec_permits_to_acquire(&self, n: usize) -> usize { + let mut curr = WaiterState(self.state.load(Acquire)); + + loop { + if !curr.is_queued() { + assert_eq!(0, curr.permits_to_acquire()); + } + + let delta = cmp::min(n, curr.permits_to_acquire()); + let rem = curr.permits_to_acquire() - delta; + + let mut next = curr; + next.set_permits_to_acquire(rem); + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => return n - delta, + Err(actual) => curr = WaiterState(actual), + } + } + } + + /// Store the number of remaining permits needed to satisfy the waiter and + /// transition to the "QUEUED" state. + /// + /// # Returns + /// + /// `true` if the `QUEUED` bit was set as part of the transition. + fn to_queued(&self, num_permits: usize) -> bool { + let mut curr = WaiterState(self.state.load(Acquire)); + + // The waiter should **not** be waiting for any permits. + debug_assert_eq!(curr.permits_to_acquire(), 0); + + loop { + let mut next = curr; + next.set_permits_to_acquire(num_permits); + next.set_queued(); + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => { + if curr.is_queued() { + return false; + } else { + // Make sure the next pointer is null + self.next.store(ptr::null_mut(), Relaxed); + return true; + } + } + Err(actual) => curr = WaiterState(actual), + } + } + } + + /// Set the number of permits to acquire. + /// + /// This function is only called when the waiter is being inserted into the + /// wait queue. Because of this, there are no concurrent threads that can + /// modify the state and using `store` is safe. + fn set_permits_to_acquire(&self, num_permits: usize) { + debug_assert!(WaiterState(self.state.load(Acquire)).is_queued()); + + let mut state = WaiterState(QUEUED); + state.set_permits_to_acquire(num_permits); + + self.state.store(state.0, Release); + } + + /// Assign permits to the waiter. + /// + /// Returns `true` if the waiter should be removed from the queue + fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { + let mut curr = WaiterState(self.state.load(Acquire)); + + loop { + let mut next = curr; + + // Number of permits to assign to this waiter + let assign = cmp::min(curr.permits_to_acquire(), *n); + + // Assign the permits + next.set_permits_to_acquire(curr.permits_to_acquire() - assign); + + if closed { + next.set_closed(); + } + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => { + // Update `n` + *n -= assign; + + if next.permits_to_acquire() == 0 { + if curr.permits_to_acquire() > 0 { + self.waker.wake(); + } + + return true; + } else { + return false; + } + } + Err(actual) => curr = WaiterState(actual), + } + } + } + + fn revert_to_idle(&self) { + // An idle node is not waiting on any permits + self.state.store(0, Relaxed); + } + + fn store_next(&self, next: NonNull) { + self.next.store(next.as_ptr(), Release); + } +} + +// ===== impl SemState ===== + +impl SemState { + /// Returns a new default `State` value. + fn new(permits: usize, stub: &Waiter) -> SemState { + assert!(permits <= MAX_PERMITS); + + if permits > 0 { + SemState((permits << NUM_SHIFT) | NUM_FLAG) + } else { + SemState(stub as *const _ as usize) + } + } + + /// Returns a `State` tracking `ptr` as the tail of the queue. + fn new_ptr(tail: NonNull, closed: bool) -> SemState { + let mut val = tail.as_ptr() as usize; + + if closed { + val |= CLOSED_FLAG; + } + + SemState(val) + } + + /// Returns the amount of remaining capacity + fn available_permits(self) -> usize { + if !self.has_available_permits() { + return 0; + } + + self.0 >> NUM_SHIFT + } + + /// Returns `true` if the state has permits that can be claimed by a waiter. + fn has_available_permits(self) -> bool { + self.0 & NUM_FLAG == NUM_FLAG + } + + fn has_waiter(self, stub: &Waiter) -> bool { + !self.has_available_permits() && !self.is_stub(stub) + } + + /// Tries to atomically acquire specified number of permits. + /// + /// # Return + /// + /// Returns `true` if the specified number of permits were acquired, `false` + /// otherwise. Returning false does not mean that there are no more + /// available permits. + fn acquire_permits(&mut self, num: usize, stub: &Waiter) -> bool { + debug_assert!(num > 0); + + if self.available_permits() < num { + return false; + } + + debug_assert!(self.waiter().is_none()); + + self.0 -= num << NUM_SHIFT; + + if self.0 == NUM_FLAG { + // Set the state to the stub pointer. + self.0 = stub as *const _ as usize; + } + + true + } + + /// Releases permits + /// + /// Returns `true` if the permits were accepted. + fn release_permits(&mut self, permits: usize, stub: &Waiter) { + debug_assert!(permits > 0); + + if self.is_stub(stub) { + self.0 = (permits << NUM_SHIFT) | NUM_FLAG | (self.0 & CLOSED_FLAG); + return; + } + + debug_assert!(self.has_available_permits()); + + self.0 += permits << NUM_SHIFT; + } + + fn is_waiter(self) -> bool { + self.0 & NUM_FLAG == 0 + } + + /// Returns the waiter, if one is set. + fn waiter(self) -> Option> { + if self.is_waiter() { + let waiter = NonNull::new(self.as_ptr()).expect("null pointer stored"); + + Some(waiter) + } else { + None + } + } + + /// Assumes `self` represents a pointer + fn as_ptr(self) -> *mut Waiter { + (self.0 & !CLOSED_FLAG) as *mut Waiter + } + + /// Sets to a pointer to a waiter. + /// + /// This can only be done from the full state. + fn set_waiter(&mut self, waiter: NonNull) { + let waiter = waiter.as_ptr() as usize; + debug_assert!(!self.is_closed()); + + self.0 = waiter; + } + + fn is_stub(self, stub: &Waiter) -> bool { + self.as_ptr() as usize == stub as *const _ as usize + } + + /// Loads the state from an AtomicUsize. + fn load(cell: &AtomicUsize, ordering: Ordering) -> SemState { + let value = cell.load(ordering); + SemState(value) + } + + fn fetch_set_closed(cell: &AtomicUsize, ordering: Ordering) -> SemState { + let value = cell.fetch_or(CLOSED_FLAG, ordering); + SemState(value) + } + + fn is_closed(self) -> bool { + self.0 & CLOSED_FLAG == CLOSED_FLAG + } + + /// Converts the state into a `usize` representation. + fn to_usize(self) -> usize { + self.0 + } +} + +impl fmt::Debug for SemState { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut fmt = fmt.debug_struct("SemState"); + + if self.is_waiter() { + fmt.field("state", &""); + } else { + fmt.field("permits", &self.available_permits()); + } + + fmt.finish() + } +} + +// ===== impl WaiterState ===== + +impl WaiterState { + fn permits_to_acquire(self) -> usize { + self.0 >> PERMIT_SHIFT + } + + fn set_permits_to_acquire(&mut self, val: usize) { + self.0 = (val << PERMIT_SHIFT) | (self.0 & !PERMIT_MASK) + } + + fn is_queued(self) -> bool { + self.0 & QUEUED == QUEUED + } + + fn set_queued(&mut self) { + self.0 |= QUEUED; + } + + fn is_closed(self) -> bool { + self.0 & CLOSED == CLOSED + } + + fn set_closed(&mut self) { + self.0 |= CLOSED; + } + + fn unset_queued(&mut self) { + assert!(self.is_queued()); + self.0 -= QUEUED; + } + + fn is_dropped(self) -> bool { + self.0 & DROPPED == DROPPED + } +} diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 0ee6b5adee7..e3cf5cc8cc7 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -104,10 +104,10 @@ impl LinkedList { /// Borrows the last element and returns it, or `None` if the list is empty. /// /// The function is safe as the lifetime of the entry is bound to `&self`. - pub(crate) fn last_mut(&self) -> Option> { + pub(crate) fn last_mut(&self) -> Option> { unsafe { let last = self.tail.as_ref()?.as_ref(); - let val = &mut *last.data.get(); + let val = &*last.data.get(); Some(Pin::new_unchecked(val)) } } From 9ae6fba51e7fe8f5d20a0861c5687df2c4ea0dad Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 19 Feb 2020 16:31:04 -0800 Subject: [PATCH 06/91] works but is horrible Signed-off-by: Eliza Weisman --- tokio/src/sync/mpsc/bounded.rs | 23 +++-- tokio/src/sync/mpsc/chan.rs | 22 ++-- tokio/src/sync/mutex.rs | 25 ++--- tokio/src/sync/rwlock.rs | 14 ++- tokio/src/sync/semaphore.rs | 13 +-- tokio/src/sync/semaphore_ll.rs | 69 +++++++++---- tokio/src/sync/tests/semaphore_ll.rs | 148 +++++++++++++-------------- tokio/src/util/linked_list.rs | 2 +- tokio/tests/sync_mpsc.rs | 65 ++++++------ 9 files changed, 213 insertions(+), 168 deletions(-) diff --git a/tokio/src/sync/mpsc/bounded.rs b/tokio/src/sync/mpsc/bounded.rs index b95611d8854..2535de8e913 100644 --- a/tokio/src/sync/mpsc/bounded.rs +++ b/tokio/src/sync/mpsc/bounded.rs @@ -5,11 +5,14 @@ use crate::sync::semaphore_ll as semaphore; use std::fmt; use std::task::{Context, Poll}; -/// Send values to the associated `Receiver`. -/// -/// Instances are created by the [`channel`](channel) function. -pub struct Sender { - chan: chan::Tx, +pin_project_lite::pin_project! { + /// Send values to the associated `Receiver`. + /// + /// Instances are created by the [`channel`](channel) function. + pub struct Sender { + #[pin] + chan: chan::Tx, + } } impl Clone for Sender { @@ -193,7 +196,9 @@ impl Sender { #[doc(hidden)] // TODO: document pub fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.chan.poll_ready(cx).map_err(|_| ClosedError::new()) + unsafe { std::pin::Pin::new_unchecked(&mut self.chan) } + .poll_ready(cx) + .map_err(|_| ClosedError::new()) } /// Attempts to immediately send a message on this `Sender` @@ -310,8 +315,10 @@ impl Sender { /// ``` pub async fn send(&mut self, value: T) -> Result<(), SendError> { use crate::future::poll_fn; - - if poll_fn(|cx| self.poll_ready(cx)).await.is_err() { + if poll_fn(|cx| unsafe { std::pin::Pin::new_unchecked(&mut self.chan) }.poll_ready(cx)) + .await + .is_err() + { return Err(SendError(value)); } diff --git a/tokio/src/sync/mpsc/chan.rs b/tokio/src/sync/mpsc/chan.rs index 2fc915d0e83..bccbef38f97 100644 --- a/tokio/src/sync/mpsc/chan.rs +++ b/tokio/src/sync/mpsc/chan.rs @@ -6,6 +6,7 @@ use crate::sync::mpsc::error::{ClosedError, TryRecvError}; use crate::sync::mpsc::{error, list}; use std::fmt; +use std::pin::Pin; use std::process; use std::sync::atomic::Ordering::{AcqRel, Relaxed}; use std::task::Poll::{Pending, Ready}; @@ -84,7 +85,7 @@ pub(crate) trait Semaphore { fn poll_acquire( &self, cx: &mut Context<'_>, - permit: &mut Self::Permit, + permit: Pin<&mut Self::Permit>, ) -> Poll>; fn try_acquire(&self, permit: &mut Self::Permit) -> Result<(), TrySendError>; @@ -186,8 +187,15 @@ where } } - pub(crate) fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.inner.semaphore.poll_acquire(cx, &mut self.permit) + pub(crate) fn poll_ready( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + let (inner, permit) = unsafe { + let this = self.get_unchecked_mut(); + (&this.inner, Pin::new_unchecked(&mut this.permit)) + }; + inner.semaphore.poll_acquire(cx, permit) } /// Send a message and notify the receiver. @@ -422,7 +430,7 @@ impl Semaphore for (crate::sync::semaphore_ll::Semaphore, usize) { fn poll_acquire( &self, cx: &mut Context<'_>, - permit: &mut Permit, + permit: Pin<&mut Permit>, ) -> Poll> { permit .poll_acquire(cx, 1, &self.0) @@ -435,7 +443,7 @@ impl Semaphore for (crate::sync::semaphore_ll::Semaphore, usize) { } fn forget(&self, permit: &mut Self::Permit) { - permit.forget(1); + permit.forget(1, &self.0); } fn close(&self) { @@ -471,9 +479,9 @@ impl Semaphore for AtomicUsize { fn poll_acquire( &self, _cx: &mut Context<'_>, - permit: &mut (), + permit: Pin<&mut ()>, ) -> Poll> { - Ready(self.try_acquire(permit).map_err(|_| ClosedError::new())) + Ready(self.try_acquire(&mut ()).map_err(|_| ClosedError::new())) } fn try_acquire(&self, _permit: &mut ()) -> Result<(), TrySendError> { diff --git a/tokio/src/sync/mutex.rs b/tokio/src/sync/mutex.rs index 2b1e8a51544..827c3732d9c 100644 --- a/tokio/src/sync/mutex.rs +++ b/tokio/src/sync/mutex.rs @@ -152,20 +152,17 @@ impl Mutex { /// A future that resolves on acquiring the lock and returns the `MutexGuard`. pub async fn lock(&self) -> MutexGuard<'_, T> { - let permit = semaphore::Permit::new(); - let pinned = permit; - pin!(pinned); - poll_fn(|cx| pinned.poll_acquire(cx, 1, &self.s)) - .await - .unwrap_or_else(|_| { - // The semaphore was closed. but, we never explicitly close it, and we have a - // handle to it through the Arc, which means that this can never happen. - unreachable!() - }); - MutexGuard { - lock: self, - permit, - } + let mut permit = semaphore::Permit::new(); + poll_fn(|cx| { + unsafe { std::pin::Pin::new_unchecked(&mut permit) }.poll_acquire(cx, 1, &self.s) + }) + .await + .unwrap_or_else(|_| { + // The semaphore was closed. but, we never explicitly close it, and we have a + // handle to it through the Arc, which means that this can never happen. + unreachable!() + }); + MutexGuard { lock: self, permit } } /// Tries to acquire the lock diff --git a/tokio/src/sync/rwlock.rs b/tokio/src/sync/rwlock.rs index b859813a772..446ecf792ea 100644 --- a/tokio/src/sync/rwlock.rs +++ b/tokio/src/sync/rwlock.rs @@ -2,6 +2,7 @@ use crate::future::poll_fn; use crate::sync::semaphore_ll::{AcquireError, Permit, Semaphore}; use std::cell::UnsafeCell; use std::ops; +use std::pin::Pin; use std::task::{Context, Poll}; #[cfg(not(loom))] @@ -109,11 +110,15 @@ struct ReleasingPermit<'a, T> { impl<'a, T> ReleasingPermit<'a, T> { fn poll_acquire( - &mut self, + self: Pin<&mut Self>, cx: &mut Context<'_>, s: &Semaphore, ) -> Poll> { - self.permit.poll_acquire(cx, self.num_permits, s) + let (num_permits, permit) = unsafe { + let this = self.get_unchecked_mut(); + (this.num_permits, Pin::new_unchecked(&mut this.permit)) + }; + permit.poll_acquire(cx, num_permits, s) } } @@ -181,8 +186,7 @@ impl RwLock { permit: Permit::new(), lock: self, }; - - poll_fn(|cx| permit.poll_acquire(cx, &self.s)) + poll_fn(|cx| unsafe { Pin::new_unchecked(&mut permit) }.poll_acquire(cx, &self.s)) .await .unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and we have a @@ -221,7 +225,7 @@ impl RwLock { lock: self, }; - poll_fn(|cx| permit.poll_acquire(cx, &self.s)) + poll_fn(|cx| unsafe { Pin::new_unchecked(&mut permit) }.poll_acquire(cx, &self.s)) .await .unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and we have a diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index a3eec95d80e..cd478901707 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -1,5 +1,6 @@ use super::semaphore_ll as ll; // low level implementation use crate::future::poll_fn; +use std::pin::Pin; /// Counting semaphore performing asynchronous permit aquisition. /// @@ -57,11 +58,11 @@ impl Semaphore { /// Acquires permit from the semaphore pub async fn acquire(&self) -> SemaphorePermit<'_> { let mut ll_permit = ll::Permit::new(); - let pinned = ll_permit; - pin!(pinned); - poll_fn(|cx| pinned.poll_acquire(cx, 1, &self.ll_sem)) - .await - .unwrap(); + poll_fn(|cx| { + unsafe { Pin::new_unchecked(&mut ll_permit) }.poll_acquire(cx, 1, &self.ll_sem) + }) + .await + .unwrap(); SemaphorePermit { sem: &self, ll_permit, @@ -86,7 +87,7 @@ impl<'a> SemaphorePermit<'a> { /// This can be used to reduce the amount of permits available from a /// semaphore. pub fn forget(mut self) { - self.ll_permit.forget(1); + self.ll_permit.forget(1, &self.sem.ll_sem); } } diff --git a/tokio/src/sync/semaphore_ll.rs b/tokio/src/sync/semaphore_ll.rs index 28dea85a7f5..73390e684b8 100644 --- a/tokio/src/sync/semaphore_ll.rs +++ b/tokio/src/sync/semaphore_ll.rs @@ -6,8 +6,7 @@ use crate::loom::{ use crate::util::linked_list::{self, LinkedList}; use std::{ - cmp, - fmt, + cmp, fmt, pin::Pin, ptr::NonNull, sync::atomic::Ordering, @@ -34,7 +33,7 @@ pub(crate) enum TryAcquireError { pub(crate) struct AcquireError(()); pin_project_lite::pin_project! { - #[derive(Debug)] + // #[derive(Debug)] pub(crate) struct Permit { #[pin] node: linked_list::Entry, @@ -114,7 +113,6 @@ impl Semaphore { fn add_permits_locked(&self, mut rem: usize, mut closed: bool) { while rem > 0 || closed { - // Release the permits and notify let mut waiters = self.waiters.lock().unwrap(); while rem > 0 { @@ -150,7 +148,7 @@ impl Semaphore { &self, cx: &mut Context<'_>, needed: u16, - node: Pin<&mut Waiter>, + node: Pin<&mut linked_list::Entry>, ) -> Poll> { let mut curr = self.permits.load(Ordering::Acquire); let remaining = loop { @@ -174,7 +172,10 @@ impl Semaphore { assert!(node.is_unlinked()); { let mut waiter = unsafe { &*node.get() }; - waiter.state.compare_exchange(0, remaining, Ordering::Release).expect("unlinked node should not want permits!"); + waiter + .state + .compare_exchange(0, remaining as usize, Ordering::AcqRel, Ordering::Acquire) + .expect("unlinked node should not want permits!"); waiter.waker.register_by_ref(cx.waker()); } @@ -187,6 +188,21 @@ impl Semaphore { } } +impl Drop for Semaphore { + fn drop(&mut self) { + self.close(); + } +} + +impl fmt::Debug for Semaphore { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("Semaphore") + .field("permits", &self.add_lock.load(Ordering::Relaxed)) + .field("add_lock", &self.add_lock.load(Ordering::Relaxed)) + .finish() + } +} + impl Permit { /// Creates a new `Permit`. /// @@ -197,7 +213,7 @@ impl Permit { Permit { node: linked_list::Entry::new(Waiter { waker: AtomicWaker::new(), - needed: 0, + state: AtomicUsize::new(0), }), state: Acquired(0), } @@ -211,7 +227,6 @@ impl Permit { } } - /// Tries to acquire the permit. If no permits are available, the current task /// is notified once a new permit becomes available. pub(crate) fn poll_acquire( @@ -254,16 +269,14 @@ impl Permit { match self.state { Waiting(requested) => { let n = cmp::min(n, requested); - + let node = unsafe { &*self.node.get() }; // Decrement - let acquired = self - .node - .try_dec_permits_to_acquire(n as usize) as u16; + let acquired = node.try_dec_permits_to_acquire(n as usize) as u16; if n == requested { self.state = Acquired(0); - // TODO: rm from wait list here! - semaphore + // // TODO: rm from wait list here! + // semaphore } else if acquired == requested - n { self.state = Waiting(acquired); } else { @@ -281,6 +294,14 @@ impl Permit { } } +impl fmt::Debug for Permit { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("Permit") + .field("state", &self.state) + .finish() + } +} + impl Default for Permit { fn default() -> Self { Self::new() @@ -298,7 +319,10 @@ impl Waiter { // Number of permits to assign to this waiter let assign = cmp::min(curr, *n); let next = curr - assign; - match self.state.compare_exchange(curr, next, Ordering::AcqRel, Ordering::Acquire) { + match self + .state + .compare_exchange(curr, next, Ordering::AcqRel, Ordering::Acquire) + { Ok(_) => { // Update `n` *n -= assign; @@ -326,14 +350,17 @@ impl Waiter { let mut curr = self.state.load(Ordering::Acquire); loop { - if !curr.is_queued() { - assert_eq!(0, curr.permits_to_acquire()); - } + // if !curr.is_queued() { + // assert_eq!(0, curr.permits_to_acquire()); + // } - let delta = cmp::min(n, curr.permits_to_acquire()); - let rem = curr.permits_to_acquire() - delta; + let delta = cmp::min(n, curr); + let rem = curr - delta; - match self.state.compare_exchange(curr, rem, Ordering::AcqRel, Ordering::Acquire) { + match self + .state + .compare_exchange(curr, rem, Ordering::AcqRel, Ordering::Acquire) + { Ok(_) => return n - delta, Err(actual) => curr = actual, } diff --git a/tokio/src/sync/tests/semaphore_ll.rs b/tokio/src/sync/tests/semaphore_ll.rs index bfb075780bb..918db6a49ff 100644 --- a/tokio/src/sync/tests/semaphore_ll.rs +++ b/tokio/src/sync/tests/semaphore_ll.rs @@ -7,15 +7,15 @@ fn poll_acquire_one_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); assert!(!permit.is_acquired()); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 99); assert!(permit.is_acquired()); // Polling again on the same waiter does not claim a new permit - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 99); assert!(permit.is_acquired()); } @@ -26,24 +26,24 @@ fn poll_acquire_many_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); assert!(!permit.is_acquired()); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 5, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); assert_eq!(s.available_permits(), 95); assert!(permit.is_acquired()); // Polling again on the same waiter does not claim a new permit - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 95); assert!(permit.is_acquired()); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 5, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); assert_eq!(s.available_permits(), 95); assert!(permit.is_acquired()); // Polling for a larger number of permits acquires more - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 8, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 8, &s))); assert_eq!(s.available_permits(), 92); assert!(permit.is_acquired()); } @@ -90,23 +90,23 @@ fn try_acquire_many_available() { fn poll_acquire_one_unavailable() { let s = Semaphore::new(1); - let mut permit_1 = task::spawn(Permit::new()); - let mut permit_2 = task::spawn(Permit::new()); + let mut permit_1 = task::spawn(Box::new(Permit::new())); + let mut permit_2 = task::spawn(Box::new(Permit::new())); // Acquire the first permit - assert_ready_ok!(permit_1.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit_1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 0); permit_2.enter(|cx, mut p| { // Try to acquire the second permit - assert_pending!(p.poll_acquire(cx, 1, &s)); + assert_pending!(p.as_mut().poll_acquire(cx, 1, &s)); }); permit_1.release(1, &s); assert_eq!(s.available_permits(), 0); assert!(permit_2.is_woken()); - assert_ready_ok!(permit_2.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit_2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); permit_2.release(1, &s); assert_eq!(s.available_permits(), 1); @@ -117,9 +117,9 @@ fn forget_acquired() { let s = Semaphore::new(1); // Polling for a permit succeeds immediately - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 0); @@ -132,9 +132,9 @@ fn forget_waiting() { let s = Semaphore::new(0); // Polling for a permit succeeds immediately - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 0); @@ -150,31 +150,31 @@ fn forget_waiting() { fn poll_acquire_many_unavailable() { let s = Semaphore::new(5); - let mut permit_1 = task::spawn(Permit::new()); - let mut permit_2 = task::spawn(Permit::new()); - let mut permit_3 = task::spawn(Permit::new()); + let mut permit_1 = task::spawn(Box::new(Permit::new())); + let mut permit_2 = task::spawn(Box::new(Permit::new())); + let mut permit_3 = task::spawn(Box::new(Permit::new())); // Acquire the first permit - assert_ready_ok!(permit_1.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit_1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 4); permit_2.enter(|cx, mut p| { // Try to acquire the second permit - assert_pending!(p.poll_acquire(cx, 5, &s)); + assert_pending!(p.as_mut().poll_acquire(cx, 5, &s)); }); assert_eq!(s.available_permits(), 0); permit_3.enter(|cx, mut p| { // Try to acquire the third permit - assert_pending!(p.poll_acquire(cx, 3, &s)); + assert_pending!(p.as_mut().poll_acquire(cx, 3, &s)); }); permit_1.release(1, &s); assert_eq!(s.available_permits(), 0); assert!(permit_2.is_woken()); - assert_ready_ok!(permit_2.enter(|cx, mut p| p.poll_acquire(cx, 5, &s))); + assert_ready_ok!(permit_2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); assert!(!permit_3.is_woken()); assert_eq!(s.available_permits(), 0); @@ -186,7 +186,7 @@ fn poll_acquire_many_unavailable() { permit_2.release(2, &s); assert!(permit_3.is_woken()); - assert_ready_ok!(permit_3.enter(|cx, mut p| p.poll_acquire(cx, 3, &s))); + assert_ready_ok!(permit_3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); } #[test] @@ -241,17 +241,17 @@ fn poll_acquire_one_zero_permits() { let s = Semaphore::new(0); assert_eq!(s.available_permits(), 0); - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); // Try to acquire the permit permit.enter(|cx, mut p| { - assert_pending!(p.poll_acquire(cx, 1, &s)); + assert_pending!(p.as_mut().poll_acquire(cx, 1, &s)); }); s.add_permits(1); assert!(permit.is_woken()); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); } #[test] @@ -268,52 +268,52 @@ fn close_semaphore_prevents_acquire() { assert_eq!(5, s.available_permits()); - let mut permit_1 = task::spawn(Permit::new()); - let mut permit_2 = task::spawn(Permit::new()); + let mut permit_1 = task::spawn(Box::new(Permit::new())); + let mut permit_2 = task::spawn(Box::new(Permit::new())); - assert_ready_err!(permit_1.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_err!(permit_1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(5, s.available_permits()); - assert_ready_err!(permit_2.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_ready_err!(permit_2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); assert_eq!(5, s.available_permits()); } #[test] fn close_semaphore_notifies_permit1() { let s = Semaphore::new(0); - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); s.close(); assert!(permit.is_woken()); - assert_ready_err!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_err!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); } #[test] fn close_semaphore_notifies_permit2() { let s = Semaphore::new(2); - let mut permit1 = task::spawn(Permit::new()); - let mut permit2 = task::spawn(Permit::new()); - let mut permit3 = task::spawn(Permit::new()); - let mut permit4 = task::spawn(Permit::new()); + let mut permit1 = task::spawn(Box::new(Permit::new())); + let mut permit2 = task::spawn(Box::new(Permit::new())); + let mut permit3 = task::spawn(Box::new(Permit::new())); + let mut permit4 = task::spawn(Box::new(Permit::new())); // Acquire a couple of permits - assert_ready_ok!(permit1.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); - assert_ready_ok!(permit2.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_pending!(permit3.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); - assert_pending!(permit4.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_pending!(permit3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_pending!(permit4.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); s.close(); assert!(permit3.is_woken()); assert!(permit4.is_woken()); - assert_ready_err!(permit3.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); - assert_ready_err!(permit4.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_err!(permit3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_ready_err!(permit4.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(0, s.available_permits()); @@ -321,7 +321,7 @@ fn close_semaphore_notifies_permit2() { assert_eq!(1, s.available_permits()); - assert_ready_err!(permit1.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_err!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); permit2.release(1, &s); @@ -332,10 +332,10 @@ fn close_semaphore_notifies_permit2() { fn poll_acquire_additional_permits_while_waiting_before_assigned() { let s = Semaphore::new(1); - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 3, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); s.add_permits(1); assert!(!permit.is_woken()); @@ -343,16 +343,16 @@ fn poll_acquire_additional_permits_while_waiting_before_assigned() { s.add_permits(1); assert!(permit.is_woken()); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 3, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); } #[test] fn try_acquire_additional_permits_while_waiting_before_assigned() { let s = Semaphore::new(1); - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); assert_err!(permit.enter(|_, mut p| p.try_acquire(3, &s))); @@ -366,45 +366,45 @@ fn try_acquire_additional_permits_while_waiting_before_assigned() { fn poll_acquire_additional_permits_while_waiting_after_assigned_success() { let s = Semaphore::new(1); - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); s.add_permits(2); assert!(permit.is_woken()); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 3, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); } #[test] fn poll_acquire_additional_permits_while_waiting_after_assigned_requeue() { let s = Semaphore::new(1); - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); s.add_permits(2); assert!(permit.is_woken()); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 4, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); s.add_permits(1); assert!(permit.is_woken()); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 4, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); } #[test] fn poll_acquire_fewer_permits_while_waiting() { let s = Semaphore::new(1); - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); assert_eq!(s.available_permits(), 0); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 0); } @@ -412,40 +412,40 @@ fn poll_acquire_fewer_permits_while_waiting() { fn poll_acquire_fewer_permits_after_assigned() { let s = Semaphore::new(1); - let mut permit1 = task::spawn(Permit::new()); - let mut permit2 = task::spawn(Permit::new()); + let mut permit1 = task::spawn(Box::new(Permit::new())); + let mut permit2 = task::spawn(Box::new(Permit::new())); - assert_pending!(permit1.enter(|cx, mut p| p.poll_acquire(cx, 5, &s))); + assert_pending!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); assert_eq!(s.available_permits(), 0); - assert_pending!(permit2.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_pending!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); s.add_permits(4); assert!(permit1.is_woken()); assert!(!permit2.is_woken()); - assert_ready_ok!(permit1.enter(|cx, mut p| p.poll_acquire(cx, 3, &s))); + assert_ready_ok!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); assert!(permit2.is_woken()); assert_eq!(s.available_permits(), 1); - assert_ready_ok!(permit2.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); } #[test] fn forget_partial_1() { let s = Semaphore::new(0); - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); s.add_permits(1); assert_eq!(0, s.available_permits()); permit.release(1, &s); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 0); } @@ -454,9 +454,9 @@ fn forget_partial_1() { fn forget_partial_2() { let s = Semaphore::new(0); - let mut permit = task::spawn(Permit::new()); + let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); s.add_permits(1); assert_eq!(0, s.available_permits()); @@ -465,6 +465,6 @@ fn forget_partial_2() { s.add_permits(1); - assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); assert_eq!(s.available_permits(), 0); } diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index e3cf5cc8cc7..5e828ef096d 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -104,7 +104,7 @@ impl LinkedList { /// Borrows the last element and returns it, or `None` if the list is empty. /// /// The function is safe as the lifetime of the entry is bound to `&self`. - pub(crate) fn last_mut(&self) -> Option> { + pub(crate) fn last(&self) -> Option> { unsafe { let last = self.tail.as_ref()?.as_ref(); let val = &*last.data.get(); diff --git a/tokio/tests/sync_mpsc.rs b/tokio/tests/sync_mpsc.rs index 7e5c60e2e2f..ab8ab386c6d 100644 --- a/tokio/tests/sync_mpsc.rs +++ b/tokio/tests/sync_mpsc.rs @@ -15,30 +15,31 @@ trait AssertSend: Send {} impl AssertSend for mpsc::Sender {} impl AssertSend for mpsc::Receiver {} -#[test] -fn send_recv_with_buffer() { - let (tx, rx) = mpsc::channel::(16); - let mut tx = task::spawn(tx); - let mut rx = task::spawn(rx); +// #[test] +// fn send_recv_with_buffer() { +// let (tx, rx) = mpsc::channel::(16); +// let mut tx = task::spawn(tx); +// let mut rx = task::spawn(rx); - // Using poll_ready / try_send - assert_ready_ok!(tx.enter(|cx, mut tx| tx.poll_ready(cx))); - tx.try_send(1).unwrap(); +// // Using poll_ready / try_send +// assert_ready_ok!(tx.enter(|cx, mut tx| tx.poll_ready(cx))); +// // tx.try_send(1).unwrap(); +// unimplemented!(); - // Without poll_ready - tx.try_send(2).unwrap(); +// // Without poll_ready +// // tx.try_send(2).unwrap(); - drop(tx); +// drop(tx); - let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); - assert_eq!(val, Some(1)); +// let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); +// assert_eq!(val, Some(1)); - let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); - assert_eq!(val, Some(2)); +// let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); +// assert_eq!(val, Some(2)); - let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); - assert!(val.is_none()); -} +// let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); +// assert!(val.is_none()); +// } #[tokio::test] async fn send_recv_stream_with_buffer() { @@ -161,23 +162,23 @@ async fn send_recv_stream_unbounded() { assert_eq!(None, rx.next().await); } -#[test] -fn no_t_bounds_buffer() { - struct NoImpls; +// #[test] +// fn no_t_bounds_buffer() { +// struct NoImpls; - let mut t1 = task::spawn(()); - let (tx, mut rx) = mpsc::channel(100); +// let mut t1 = task::spawn(()); +// let (tx, mut rx) = mpsc::channel(100); - // sender should be Debug even though T isn't Debug - println!("{:?}", tx); - // same with Receiver - println!("{:?}", rx); - // and sender should be Clone even though T isn't Clone - assert!(tx.clone().try_send(NoImpls).is_ok()); +// // sender should be Debug even though T isn't Debug +// println!("{:?}", tx); +// // same with Receiver +// println!("{:?}", rx); +// // and sender should be Clone even though T isn't Clone +// assert!(tx.clone().try_send(NoImpls).is_ok()); - let val = assert_ready!(t1.enter(|cx, _| rx.poll_recv(cx))); - assert!(val.is_some()); -} +// let val = assert_ready!(t1.enter(|cx, _| rx.poll_recv(cx))); +// assert!(val.is_some()); +// } #[test] fn no_t_bounds_unbounded() { From ea499fe72dc36281b8228fed5c3ab0246994af42 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 20 Feb 2020 17:23:34 -0800 Subject: [PATCH 07/91] wip Signed-off-by: Eliza Weisman --- tokio/src/sync/mpsc/chan.rs | 7 +- tokio/src/sync/mutex.rs | 6 +- tokio/src/sync/rwlock.rs | 36 +- tokio/src/sync/semaphore.rs | 6 +- tokio/src/sync/semaphore_ll.rs | 229 ++++++++---- tokio/src/sync/tests/semaphore_ll.rs | 501 ++++++++++++++------------- 6 files changed, 432 insertions(+), 353 deletions(-) diff --git a/tokio/src/sync/mpsc/chan.rs b/tokio/src/sync/mpsc/chan.rs index bccbef38f97..4151cf787cd 100644 --- a/tokio/src/sync/mpsc/chan.rs +++ b/tokio/src/sync/mpsc/chan.rs @@ -432,9 +432,10 @@ impl Semaphore for (crate::sync::semaphore_ll::Semaphore, usize) { cx: &mut Context<'_>, permit: Pin<&mut Permit>, ) -> Poll> { - permit - .poll_acquire(cx, 1, &self.0) - .map_err(|_| ClosedError::new()) + // permit + // .poll_acquire(cx, 1, &self.0) + // .map_err(|_| ClosedError::new()) + unimplemented!() } fn try_acquire(&self, permit: &mut Permit) -> Result<(), TrySendError> { diff --git a/tokio/src/sync/mutex.rs b/tokio/src/sync/mutex.rs index 827c3732d9c..dc13d19fdaa 100644 --- a/tokio/src/sync/mutex.rs +++ b/tokio/src/sync/mutex.rs @@ -153,11 +153,7 @@ impl Mutex { /// A future that resolves on acquiring the lock and returns the `MutexGuard`. pub async fn lock(&self) -> MutexGuard<'_, T> { let mut permit = semaphore::Permit::new(); - poll_fn(|cx| { - unsafe { std::pin::Pin::new_unchecked(&mut permit) }.poll_acquire(cx, 1, &self.s) - }) - .await - .unwrap_or_else(|_| { + permit.acquire(1, &self.s).await.unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and we have a // handle to it through the Arc, which means that this can never happen. unreachable!() diff --git a/tokio/src/sync/rwlock.rs b/tokio/src/sync/rwlock.rs index 446ecf792ea..52fc0edb273 100644 --- a/tokio/src/sync/rwlock.rs +++ b/tokio/src/sync/rwlock.rs @@ -109,16 +109,8 @@ struct ReleasingPermit<'a, T> { } impl<'a, T> ReleasingPermit<'a, T> { - fn poll_acquire( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - s: &Semaphore, - ) -> Poll> { - let (num_permits, permit) = unsafe { - let this = self.get_unchecked_mut(); - (this.num_permits, Pin::new_unchecked(&mut this.permit)) - }; - permit.poll_acquire(cx, num_permits, s) + async fn acquire(&mut self) -> Result<(), AcquireError> { + self.permit.acquire(self.num_permits, &self.lock.s).await } } @@ -186,13 +178,11 @@ impl RwLock { permit: Permit::new(), lock: self, }; - poll_fn(|cx| unsafe { Pin::new_unchecked(&mut permit) }.poll_acquire(cx, &self.s)) - .await - .unwrap_or_else(|_| { - // The semaphore was closed. but, we never explicitly close it, and we have a - // handle to it through the Arc, which means that this can never happen. - unreachable!() - }); + permit.acquire().await.unwrap_or_else(|_| { + // The semaphore was closed. but, we never explicitly close it, and we have a + // handle to it through the Arc, which means that this can never happen. + unreachable!() + }); RwLockReadGuard { lock: self, permit } } @@ -225,13 +215,11 @@ impl RwLock { lock: self, }; - poll_fn(|cx| unsafe { Pin::new_unchecked(&mut permit) }.poll_acquire(cx, &self.s)) - .await - .unwrap_or_else(|_| { - // The semaphore was closed. but, we never explicitly close it, and we have a - // handle to it through the Arc, which means that this can never happen. - unreachable!() - }); + permit.acquire().await.unwrap_or_else(|_| { + // The semaphore was closed. but, we never explicitly close it, and we have a + // handle to it through the Arc, which means that this can never happen. + unreachable!() + }); RwLockWriteGuard { lock: self, permit } } diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index cd478901707..df53a3311c2 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -58,11 +58,7 @@ impl Semaphore { /// Acquires permit from the semaphore pub async fn acquire(&self) -> SemaphorePermit<'_> { let mut ll_permit = ll::Permit::new(); - poll_fn(|cx| { - unsafe { Pin::new_unchecked(&mut ll_permit) }.poll_acquire(cx, 1, &self.ll_sem) - }) - .await - .unwrap(); + ll_permit.acquire(1, &self.ll_sem).await.unwrap(); SemaphorePermit { sem: &self, ll_permit, diff --git a/tokio/src/sync/semaphore_ll.rs b/tokio/src/sync/semaphore_ll.rs index 73390e684b8..5dd758cd331 100644 --- a/tokio/src/sync/semaphore_ll.rs +++ b/tokio/src/sync/semaphore_ll.rs @@ -1,12 +1,13 @@ use crate::loom::{ alloc, future::AtomicWaker, - sync::{atomic::AtomicUsize, Mutex}, + sync::{atomic::AtomicUsize, Mutex, MutexGuard}, }; use crate::util::linked_list::{self, LinkedList}; use std::{ cmp, fmt, + future::Future, pin::Pin, ptr::NonNull, sync::atomic::Ordering, @@ -32,13 +33,16 @@ pub(crate) enum TryAcquireError { #[derive(Debug)] pub(crate) struct AcquireError(()); -pin_project_lite::pin_project! { - // #[derive(Debug)] - pub(crate) struct Permit { - #[pin] - node: linked_list::Entry, - state: PermitState, - } +#[derive(Debug)] +pub(crate) struct Permit { + state: PermitState, +} + +pub(crate) struct Acquire<'a> { + node: linked_list::Entry, + semaphore: &'a Semaphore, + permit: &'a mut Permit, + num_permits: u16, } /// Permit state @@ -91,7 +95,7 @@ impl Semaphore { return; } - self.add_permits_locked(0, true); + self.add_permits_locked(0, true, self.waiters.lock().unwrap()); } /// Adds `n` new permits to the semaphore. @@ -108,19 +112,28 @@ impl Semaphore { // pending waiters. return; } - self.add_permits_locked(n, false); + + self.add_permits_locked(n, false, self.waiters.lock().unwrap()); } - fn add_permits_locked(&self, mut rem: usize, mut closed: bool) { + fn add_permits_locked( + &self, + mut rem: usize, + mut closed: bool, + mut waiters: MutexGuard<'_, LinkedList>, + ) { while rem > 0 || closed { + // how many permits are we releasing on this pass? + + let initial = rem; // Release the permits and notify - let mut waiters = self.waiters.lock().unwrap(); - while rem > 0 { + while dbg!(rem > 0) { let pop = match waiters.last() { - Some(last) => last.assign_permits(&mut rem, closed), + Some(last) => dbg!(last.assign_permits(&mut rem, closed)), None => { self.permits.fetch_add(rem, Ordering::Release); break; + // false } }; if pop { @@ -128,7 +141,7 @@ impl Semaphore { } } - let n = rem << 1; + let n = (initial - rem) << 1; let actual = if closed { let actual = self.add_lock.fetch_sub(n | 1, Ordering::AcqRel); @@ -140,7 +153,7 @@ impl Semaphore { actual }; - rem = (actual >> 1) - rem; + rem = (actual - initial) >> 1; } } @@ -151,7 +164,7 @@ impl Semaphore { node: Pin<&mut linked_list::Entry>, ) -> Poll> { let mut curr = self.permits.load(Ordering::Acquire); - let remaining = loop { + loop { if curr == CLOSED { return Ready(Err(AcquireError(()))); } @@ -162,26 +175,23 @@ impl Semaphore { Ordering::AcqRel, Ordering::Acquire, ) { - Ok(_) if curr >= needed as usize => return Ready(Ok(())), - Ok(_) => break needed - curr as u16, + Ok(_) => break, Err(actual) => curr = actual, } - }; - // if we've not returned already, then we need to push the waiter to the - // wait queue. - assert!(node.is_unlinked()); - { - let mut waiter = unsafe { &*node.get() }; - waiter - .state - .compare_exchange(0, remaining as usize, Ordering::AcqRel, Ordering::Acquire) - .expect("unlinked node should not want permits!"); - waiter.waker.register_by_ref(cx.waker()); } + assert!(node.is_unlinked()); + let waiter = unsafe { &*node.get() }; + if waiter.state.fetch_sub(curr, Ordering::AcqRel) == curr { + return Ready(Ok(())); // yay! + }; + // otherwise, register the waker & enqueue the node. + waiter.waker.register_by_ref(cx.waker()); + let mut queue = self.waiters.lock().unwrap(); unsafe { queue.push_front(node); + println!("enqueue"); } Pending @@ -210,13 +220,7 @@ impl Permit { pub(crate) fn new() -> Permit { use PermitState::Acquired; - Permit { - node: linked_list::Entry::new(Waiter { - waker: AtomicWaker::new(), - state: AtomicUsize::new(0), - }), - state: Acquired(0), - } + Permit { state: Acquired(0) } } /// Returns `true` if the permit has been acquired @@ -227,15 +231,27 @@ impl Permit { } } - /// Tries to acquire the permit. If no permits are available, the current task + /// Returns a future that tries to acquire the permit. If no permits are available, the current task /// is notified once a new permit becomes available. - pub(crate) fn poll_acquire( - self: Pin<&mut Self>, - cx: &mut Context<'_>, + pub(crate) fn acquire<'a>( + &'a mut self, num_permits: u16, - semaphore: &Semaphore, - ) -> Poll> { - semaphore.poll_acquire(cx, num_permits, self.project().node) + semaphore: &'a Semaphore, + ) -> Acquire<'a> { + self.state = match self.state { + PermitState::Acquired(0) => PermitState::Waiting(num_permits), + PermitState::Waiting(n) => PermitState::Waiting(cmp::max(n, num_permits)), + PermitState::Acquired(n) => PermitState::Acquired(n), + }; + Acquire { + node: linked_list::Entry::new(Waiter { + waker: AtomicWaker::new(), + state: AtomicUsize::new(num_permits as usize), + }), + semaphore, + permit: self, + num_permits, + } } /// Tries to acquire the permit. @@ -268,40 +284,35 @@ impl Permit { match self.state { Waiting(requested) => { - let n = cmp::min(n, requested); - let node = unsafe { &*self.node.get() }; - // Decrement - let acquired = node.try_dec_permits_to_acquire(n as usize) as u16; - - if n == requested { - self.state = Acquired(0); - // // TODO: rm from wait list here! - // semaphore - } else if acquired == requested - n { - self.state = Waiting(acquired); - } else { - self.state = Waiting(requested - n); - } - - acquired + panic!( + "cannot forget permits while in wait queue; we are already borrowed mutably?" + ) + // let n = cmp::min(n, requested); + // let node = unsafe { &*self.node.get() }; + // // Decrement + // let acquired = node.try_dec_permits_to_acquire(n as usize) as u16; + + // if n == requested { + // self.state = Acquired(0); + // // // TODO: rm from wait list here! + // // semaphore + // } else if acquired == requested - n { + // self.state = Waiting(acquired); + // } else { + // self.state = Waiting(requested - n); + // } + + // acquired } Acquired(acquired) => { let n = cmp::min(n, acquired); self.state = Acquired(acquired - n); - n + dbg!(n) } } } } -impl fmt::Debug for Permit { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("Permit") - .field("state", &self.state) - .finish() - } -} - impl Default for Permit { fn default() -> Self { Self::new() @@ -309,6 +320,8 @@ impl Default for Permit { } impl Waiter { + const QUEUED: usize = 0b1; + /// Assign permits to the waiter. /// /// Returns `true` if the waiter should be removed from the queue @@ -368,6 +381,84 @@ impl Waiter { } } +impl Future for Acquire<'_> { + type Output = Result<(), AcquireError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let (node, semaphore, permit, mut needed) = self.project(); + dbg!(&semaphore, &permit, &needed); + permit.state = match permit.state { + PermitState::Acquired(n) if n >= needed => return Ready(Ok(())), + PermitState::Acquired(n) => { + ready!(semaphore.poll_acquire(cx, needed, node))?; + PermitState::Acquired(needed) + } + PermitState::Waiting(_n) => { + assert_eq!(_n, needed, "how the heck did you get in this state?"); + if unsafe { &*node.get() }.state.load(Ordering::Acquire) > 0 { + ready!(semaphore.poll_acquire(cx, needed, node))?; + } + PermitState::Acquired(needed) + } + }; + Ready(Ok(())) + } +} + +impl Acquire<'_> { + fn project( + self: Pin<&mut Self>, + ) -> ( + Pin<&mut linked_list::Entry>, + &Semaphore, + &mut Permit, + u16, + ) { + fn is_unpin() {} + unsafe { + // Safety: all fields other than `node` are `Unpin` + + is_unpin::<&Semaphore>(); + is_unpin::<&mut Permit>(); + is_unpin::(); + + let this = self.get_unchecked_mut(); + ( + Pin::new_unchecked(&mut this.node), + &this.semaphore, + &mut this.permit, + this.num_permits, + ) + } + } +} + +impl Drop for Acquire<'_> { + fn drop(&mut self) { + dbg!("drop acquire"); + if dbg!(self.node.is_unlinked()) { + // don't need to release permits + return; + } + + // Safety: if the node is linked, then we are already pinned + let (mut node, semaphore, permit, needed) = unsafe { Pin::new_unchecked(self).project() }; + + // This is where we ensure safety. The future is being dropped, + // which means we must ensure that the waiter entry is no longer stored + // in the linked list. + let mut waiters = semaphore.waiters.lock().unwrap(); + + // remove the entry from the list + // + // safety: the waiter is only added to `waiters` by virtue of it + // being the only `LinkedList` available to the type. + unsafe { waiters.remove(node.as_mut()) }; + + // TODO(eliza): release permits to next waiter + } +} + // ===== impl AcquireError ==== impl AcquireError { diff --git a/tokio/src/sync/tests/semaphore_ll.rs b/tokio/src/sync/tests/semaphore_ll.rs index 918db6a49ff..f8a2d9f1ce6 100644 --- a/tokio/src/sync/tests/semaphore_ll.rs +++ b/tokio/src/sync/tests/semaphore_ll.rs @@ -7,15 +7,15 @@ fn poll_acquire_one_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = Permit::new(); assert!(!permit.is_acquired()); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); assert_eq!(s.available_permits(), 99); assert!(permit.is_acquired()); // Polling again on the same waiter does not claim a new permit - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); assert_eq!(s.available_permits(), 99); assert!(permit.is_acquired()); } @@ -26,24 +26,24 @@ fn poll_acquire_many_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = Permit::new(); assert!(!permit.is_acquired()); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); + assert_ready_ok!(task::spawn(permit.acquire(5, &s)).poll()); assert_eq!(s.available_permits(), 95); assert!(permit.is_acquired()); // Polling again on the same waiter does not claim a new permit - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); assert_eq!(s.available_permits(), 95); assert!(permit.is_acquired()); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); + assert_ready_ok!(task::spawn(permit.acquire(5, &s)).poll()); assert_eq!(s.available_permits(), 95); assert!(permit.is_acquired()); // Polling for a larger number of permits acquires more - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 8, &s))); + assert_ready_ok!(task::spawn(permit.acquire(8, &s)).poll()); assert_eq!(s.available_permits(), 92); assert!(permit.is_acquired()); } @@ -87,26 +87,35 @@ fn try_acquire_many_available() { } #[test] + fn poll_acquire_one_unavailable() { let s = Semaphore::new(1); - let mut permit_1 = task::spawn(Box::new(Permit::new())); - let mut permit_2 = task::spawn(Box::new(Permit::new())); + let mut permit_1 = Permit::new(); + let mut permit_2 = Permit::new(); - // Acquire the first permit - assert_ready_ok!(permit_1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_eq!(s.available_permits(), 0); + { + let mut acquire_1 = task::spawn(permit_1.acquire(1, &s)); + // Acquire the first permit + assert_ready_ok!(acquire_1.poll()); + assert_eq!(s.available_permits(), 0); + } - permit_2.enter(|cx, mut p| { + { + let mut acquire_2 = task::spawn(permit_2.acquire(1, &s)); // Try to acquire the second permit - assert_pending!(p.as_mut().poll_acquire(cx, 1, &s)); - }); + assert_pending!(acquire_2.poll()); + assert_eq!(s.available_permits(), 0); - permit_1.release(1, &s); + permit_1.release(1, &s); - assert_eq!(s.available_permits(), 0); - assert!(permit_2.is_woken()); - assert_ready_ok!(permit_2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_eq!(s.available_permits(), 0); + assert!(acquire_2.is_woken()); + assert_ready_ok!(acquire_2.poll()); + assert_eq!(s.available_permits(), 0); + } + + assert_ready_ok!(task::spawn(permit_2.acquire(1, &s)).poll()); permit_2.release(1, &s); assert_eq!(s.available_permits(), 1); @@ -117,354 +126,352 @@ fn forget_acquired() { let s = Semaphore::new(1); // Polling for a permit succeeds immediately - let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = Permit::new(); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); assert_eq!(s.available_permits(), 0); - permit.forget(1); + permit.forget(1, &s); assert_eq!(s.available_permits(), 0); } -#[test] -fn forget_waiting() { - let s = Semaphore::new(0); +// #[test] +// fn forget_waiting() { +// let s = Semaphore::new(0); - // Polling for a permit succeeds immediately - let mut permit = task::spawn(Box::new(Permit::new())); +// // Polling for a permit succeeds immediately +// let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_eq!(s.available_permits(), 0); +// assert_eq!(s.available_permits(), 0); - permit.forget(1); +// permit.forget(1); - s.add_permits(1); +// s.add_permits(1); - assert!(!permit.is_woken()); - assert_eq!(s.available_permits(), 1); -} +// assert!(!permit.is_woken()); +// assert_eq!(s.available_permits(), 1); +// } #[test] fn poll_acquire_many_unavailable() { let s = Semaphore::new(5); - let mut permit_1 = task::spawn(Box::new(Permit::new())); - let mut permit_2 = task::spawn(Box::new(Permit::new())); - let mut permit_3 = task::spawn(Box::new(Permit::new())); + let mut permit_1 = Permit::new(); + let mut permit_2 = Permit::new(); + let mut permit_3 = Permit::new(); // Acquire the first permit - assert_ready_ok!(permit_1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_ready_ok!(task::spawn(permit_1.acquire(1, &s)).poll()); assert_eq!(s.available_permits(), 4); - permit_2.enter(|cx, mut p| { - // Try to acquire the second permit - assert_pending!(p.as_mut().poll_acquire(cx, 5, &s)); - }); - + // Try to acquire the second permit + let mut acquire_2 = task::spawn(permit_2.acquire(5, &s)); + assert_pending!(acquire_2.poll()); assert_eq!(s.available_permits(), 0); - permit_3.enter(|cx, mut p| { - // Try to acquire the third permit - assert_pending!(p.as_mut().poll_acquire(cx, 3, &s)); - }); + // Try to acquire the third permit + let mut acquire_3 = task::spawn(permit_3.acquire(3, &s)); + assert_pending!(acquire_3.poll()); + // permit_3.enter(|cx, mut p| {}); - permit_1.release(1, &s); + // permit_1.release(1, &s); - assert_eq!(s.available_permits(), 0); - assert!(permit_2.is_woken()); - assert_ready_ok!(permit_2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); + // assert_eq!(s.available_permits(), 0); + // assert!(permit_2.is_woken()); + // assert_ready_ok!(permit_2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); - assert!(!permit_3.is_woken()); - assert_eq!(s.available_permits(), 0); + // assert!(!permit_3.is_woken()); + // assert_eq!(s.available_permits(), 0); - permit_2.release(1, &s); - assert!(!permit_3.is_woken()); - assert_eq!(s.available_permits(), 0); + // permit_2.release(1, &s); + // assert!(!permit_3.is_woken()); + // assert_eq!(s.available_permits(), 0); - permit_2.release(2, &s); - assert!(permit_3.is_woken()); + // permit_2.release(2, &s); + // assert!(permit_3.is_woken()); - assert_ready_ok!(permit_3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); + // assert_ready_ok!(permit_3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); } -#[test] -fn try_acquire_one_unavailable() { - let s = Semaphore::new(1); +// #[test] +// fn try_acquire_one_unavailable() { +// let s = Semaphore::new(1); - let mut permit_1 = Permit::new(); - let mut permit_2 = Permit::new(); +// let mut permit_1 = Permit::new(); +// let mut permit_2 = Permit::new(); - // Acquire the first permit - assert_ok!(permit_1.try_acquire(1, &s)); - assert_eq!(s.available_permits(), 0); +// // Acquire the first permit +// assert_ok!(permit_1.try_acquire(1, &s)); +// assert_eq!(s.available_permits(), 0); - assert_err!(permit_2.try_acquire(1, &s)); +// assert_err!(permit_2.try_acquire(1, &s)); - permit_1.release(1, &s); +// permit_1.release(1, &s); - assert_eq!(s.available_permits(), 1); - assert_ok!(permit_2.try_acquire(1, &s)); +// assert_eq!(s.available_permits(), 1); +// assert_ok!(permit_2.try_acquire(1, &s)); - permit_2.release(1, &s); - assert_eq!(s.available_permits(), 1); -} +// permit_2.release(1, &s); +// assert_eq!(s.available_permits(), 1); +// } -#[test] -fn try_acquire_many_unavailable() { - let s = Semaphore::new(5); +// #[test] +// fn try_acquire_many_unavailable() { +// let s = Semaphore::new(5); - let mut permit_1 = Permit::new(); - let mut permit_2 = Permit::new(); +// let mut permit_1 = Permit::new(); +// let mut permit_2 = Permit::new(); - // Acquire the first permit - assert_ok!(permit_1.try_acquire(1, &s)); - assert_eq!(s.available_permits(), 4); +// // Acquire the first permit +// assert_ok!(permit_1.try_acquire(1, &s)); +// assert_eq!(s.available_permits(), 4); - assert_err!(permit_2.try_acquire(5, &s)); +// assert_err!(permit_2.try_acquire(5, &s)); - permit_1.release(1, &s); - assert_eq!(s.available_permits(), 5); +// permit_1.release(1, &s); +// assert_eq!(s.available_permits(), 5); - assert_ok!(permit_2.try_acquire(5, &s)); +// assert_ok!(permit_2.try_acquire(5, &s)); - permit_2.release(1, &s); - assert_eq!(s.available_permits(), 1); +// permit_2.release(1, &s); +// assert_eq!(s.available_permits(), 1); - permit_2.release(1, &s); - assert_eq!(s.available_permits(), 2); -} +// permit_2.release(1, &s); +// assert_eq!(s.available_permits(), 2); +// } -#[test] -fn poll_acquire_one_zero_permits() { - let s = Semaphore::new(0); - assert_eq!(s.available_permits(), 0); +// #[test] +// fn poll_acquire_one_zero_permits() { +// let s = Semaphore::new(0); +// assert_eq!(s.available_permits(), 0); - let mut permit = task::spawn(Box::new(Permit::new())); +// let mut permit = task::spawn(Box::new(Permit::new())); - // Try to acquire the permit - permit.enter(|cx, mut p| { - assert_pending!(p.as_mut().poll_acquire(cx, 1, &s)); - }); +// // Try to acquire the permit +// permit.enter(|cx, mut p| { +// assert_pending!(p.as_mut().poll_acquire(cx, 1, &s)); +// }); - s.add_permits(1); +// s.add_permits(1); - assert!(permit.is_woken()); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -} +// assert!(permit.is_woken()); +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// } -#[test] -#[should_panic] -fn validates_max_permits() { - use std::usize; - Semaphore::new((usize::MAX >> 2) + 1); -} +// #[test] +// #[should_panic] +// fn validates_max_permits() { +// use std::usize; +// Semaphore::new((usize::MAX >> 2) + 1); +// } -#[test] -fn close_semaphore_prevents_acquire() { - let s = Semaphore::new(5); - s.close(); +// #[test] +// fn close_semaphore_prevents_acquire() { +// let s = Semaphore::new(5); +// s.close(); - assert_eq!(5, s.available_permits()); +// assert_eq!(5, s.available_permits()); - let mut permit_1 = task::spawn(Box::new(Permit::new())); - let mut permit_2 = task::spawn(Box::new(Permit::new())); +// let mut permit_1 = task::spawn(Box::new(Permit::new())); +// let mut permit_2 = task::spawn(Box::new(Permit::new())); - assert_ready_err!(permit_1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_eq!(5, s.available_permits()); +// assert_ready_err!(permit_1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_eq!(5, s.available_permits()); - assert_ready_err!(permit_2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - assert_eq!(5, s.available_permits()); -} +// assert_ready_err!(permit_2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// assert_eq!(5, s.available_permits()); +// } -#[test] -fn close_semaphore_notifies_permit1() { - let s = Semaphore::new(0); - let mut permit = task::spawn(Box::new(Permit::new())); +// #[test] +// fn close_semaphore_notifies_permit1() { +// let s = Semaphore::new(0); +// let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - s.close(); +// s.close(); - assert!(permit.is_woken()); - assert_ready_err!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -} +// assert!(permit.is_woken()); +// assert_ready_err!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// } -#[test] -fn close_semaphore_notifies_permit2() { - let s = Semaphore::new(2); +// #[test] +// fn close_semaphore_notifies_permit2() { +// let s = Semaphore::new(2); - let mut permit1 = task::spawn(Box::new(Permit::new())); - let mut permit2 = task::spawn(Box::new(Permit::new())); - let mut permit3 = task::spawn(Box::new(Permit::new())); - let mut permit4 = task::spawn(Box::new(Permit::new())); +// let mut permit1 = task::spawn(Box::new(Permit::new())); +// let mut permit2 = task::spawn(Box::new(Permit::new())); +// let mut permit3 = task::spawn(Box::new(Permit::new())); +// let mut permit4 = task::spawn(Box::new(Permit::new())); - // Acquire a couple of permits - assert_ready_ok!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_ready_ok!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// // Acquire a couple of permits +// assert_ready_ok!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_ready_ok!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_pending!(permit3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_pending!(permit4.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_pending!(permit3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_pending!(permit4.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - s.close(); +// s.close(); - assert!(permit3.is_woken()); - assert!(permit4.is_woken()); +// assert!(permit3.is_woken()); +// assert!(permit4.is_woken()); - assert_ready_err!(permit3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_ready_err!(permit4.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_ready_err!(permit3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_ready_err!(permit4.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_eq!(0, s.available_permits()); +// assert_eq!(0, s.available_permits()); - permit1.release(1, &s); +// permit1.release(1, &s); - assert_eq!(1, s.available_permits()); +// assert_eq!(1, s.available_permits()); - assert_ready_err!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_ready_err!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - permit2.release(1, &s); +// permit2.release(1, &s); - assert_eq!(2, s.available_permits()); -} +// assert_eq!(2, s.available_permits()); +// } -#[test] -fn poll_acquire_additional_permits_while_waiting_before_assigned() { - let s = Semaphore::new(1); +// #[test] +// fn poll_acquire_additional_permits_while_waiting_before_assigned() { +// let s = Semaphore::new(1); - let mut permit = task::spawn(Box::new(Permit::new())); +// let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); - s.add_permits(1); - assert!(!permit.is_woken()); +// s.add_permits(1); +// assert!(!permit.is_woken()); - s.add_permits(1); - assert!(permit.is_woken()); +// s.add_permits(1); +// assert!(permit.is_woken()); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); -} +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); +// } -#[test] -fn try_acquire_additional_permits_while_waiting_before_assigned() { - let s = Semaphore::new(1); +// #[test] +// fn try_acquire_additional_permits_while_waiting_before_assigned() { +// let s = Semaphore::new(1); - let mut permit = task::spawn(Box::new(Permit::new())); +// let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - assert_err!(permit.enter(|_, mut p| p.try_acquire(3, &s))); +// assert_err!(permit.enter(|_, mut p| p.try_acquire(3, &s))); - s.add_permits(1); - assert!(permit.is_woken()); +// s.add_permits(1); +// assert!(permit.is_woken()); - assert_ok!(permit.enter(|_, mut p| p.try_acquire(2, &s))); -} +// assert_ok!(permit.enter(|_, mut p| p.try_acquire(2, &s))); +// } -#[test] -fn poll_acquire_additional_permits_while_waiting_after_assigned_success() { - let s = Semaphore::new(1); +// #[test] +// fn poll_acquire_additional_permits_while_waiting_after_assigned_success() { +// let s = Semaphore::new(1); - let mut permit = task::spawn(Box::new(Permit::new())); +// let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - s.add_permits(2); +// s.add_permits(2); - assert!(permit.is_woken()); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); -} +// assert!(permit.is_woken()); +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); +// } -#[test] -fn poll_acquire_additional_permits_while_waiting_after_assigned_requeue() { - let s = Semaphore::new(1); +// #[test] +// fn poll_acquire_additional_permits_while_waiting_after_assigned_requeue() { +// let s = Semaphore::new(1); - let mut permit = task::spawn(Box::new(Permit::new())); +// let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - s.add_permits(2); +// s.add_permits(2); - assert!(permit.is_woken()); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); +// assert!(permit.is_woken()); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); - s.add_permits(1); +// s.add_permits(1); - assert!(permit.is_woken()); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); -} +// assert!(permit.is_woken()); +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); +// } -#[test] -fn poll_acquire_fewer_permits_while_waiting() { - let s = Semaphore::new(1); +// #[test] +// fn poll_acquire_fewer_permits_while_waiting() { +// let s = Semaphore::new(1); - let mut permit = task::spawn(Box::new(Permit::new())); +// let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - assert_eq!(s.available_permits(), 0); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// assert_eq!(s.available_permits(), 0); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_eq!(s.available_permits(), 0); -} +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_eq!(s.available_permits(), 0); +// } -#[test] -fn poll_acquire_fewer_permits_after_assigned() { - let s = Semaphore::new(1); +// #[test] +// fn poll_acquire_fewer_permits_after_assigned() { +// let s = Semaphore::new(1); - let mut permit1 = task::spawn(Box::new(Permit::new())); - let mut permit2 = task::spawn(Box::new(Permit::new())); +// let mut permit1 = task::spawn(Box::new(Permit::new())); +// let mut permit2 = task::spawn(Box::new(Permit::new())); - assert_pending!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); - assert_eq!(s.available_permits(), 0); +// assert_pending!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); +// assert_eq!(s.available_permits(), 0); - assert_pending!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_pending!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - s.add_permits(4); - assert!(permit1.is_woken()); - assert!(!permit2.is_woken()); +// s.add_permits(4); +// assert!(permit1.is_woken()); +// assert!(!permit2.is_woken()); - assert_ready_ok!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); +// assert_ready_ok!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); - assert!(permit2.is_woken()); - assert_eq!(s.available_permits(), 1); +// assert!(permit2.is_woken()); +// assert_eq!(s.available_permits(), 1); - assert_ready_ok!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -} +// assert_ready_ok!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// } -#[test] -fn forget_partial_1() { - let s = Semaphore::new(0); +// #[test] +// fn forget_partial_1() { +// let s = Semaphore::new(0); - let mut permit = task::spawn(Box::new(Permit::new())); +// let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - s.add_permits(1); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// s.add_permits(1); - assert_eq!(0, s.available_permits()); +// assert_eq!(0, s.available_permits()); - permit.release(1, &s); +// permit.release(1, &s); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - assert_eq!(s.available_permits(), 0); -} +// assert_eq!(s.available_permits(), 0); +// } -#[test] -fn forget_partial_2() { - let s = Semaphore::new(0); +// #[test] +// fn forget_partial_2() { +// let s = Semaphore::new(0); - let mut permit = task::spawn(Box::new(Permit::new())); +// let mut permit = task::spawn(Box::new(Permit::new())); - assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - s.add_permits(1); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// s.add_permits(1); - assert_eq!(0, s.available_permits()); +// assert_eq!(0, s.available_permits()); - permit.release(1, &s); +// permit.release(1, &s); - s.add_permits(1); +// s.add_permits(1); - assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - assert_eq!(s.available_permits(), 0); -} +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// assert_eq!(s.available_permits(), 0); +// } From 3b4a14f347332cb97a2aef1acd1b8d138225bf99 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 21 Feb 2020 15:12:07 -0800 Subject: [PATCH 08/91] fix weird logic Signed-off-by: Eliza Weisman --- tokio/src/sync/semaphore_ll.rs | 45 +++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/tokio/src/sync/semaphore_ll.rs b/tokio/src/sync/semaphore_ll.rs index 5dd758cd331..c0469a98d87 100644 --- a/tokio/src/sync/semaphore_ll.rs +++ b/tokio/src/sync/semaphore_ll.rs @@ -164,27 +164,45 @@ impl Semaphore { node: Pin<&mut linked_list::Entry>, ) -> Poll> { let mut curr = self.permits.load(Ordering::Acquire); - loop { + let needed = needed as usize; + let (acquired, remaining) = loop { + dbg!(needed, curr); if curr == CLOSED { return Ready(Err(AcquireError(()))); } - let next = curr.saturating_sub(needed as usize); + let mut remaining = 0; + let (next, acquired) = if dbg!(curr) >= dbg!(needed) { + (curr - needed, needed) + } else { + remaining = needed - curr; + (0, curr) + }; match self.permits.compare_exchange_weak( curr, next, Ordering::AcqRel, Ordering::Acquire, ) { - Ok(_) => break, + Ok(_) => break (acquired, remaining), Err(actual) => curr = actual, } + }; + dbg!(acquired, remaining); + if remaining == 0 { + return Ready(Ok(())); } assert!(node.is_unlinked()); let waiter = unsafe { &*node.get() }; - if waiter.state.fetch_sub(curr, Ordering::AcqRel) == curr { - return Ready(Ok(())); // yay! - }; + waiter + .state + .compare_exchange( + Waiter::UNQUEUED, + remaining, + Ordering::Release, + Ordering::Relaxed, + ) + .expect("not unqueued"); // otherwise, register the waker & enqueue the node. waiter.waker.register_by_ref(cx.waker()); @@ -244,10 +262,7 @@ impl Permit { PermitState::Acquired(n) => PermitState::Acquired(n), }; Acquire { - node: linked_list::Entry::new(Waiter { - waker: AtomicWaker::new(), - state: AtomicUsize::new(num_permits as usize), - }), + node: linked_list::Entry::new(Waiter::new()), semaphore, permit: self, num_permits, @@ -320,7 +335,13 @@ impl Default for Permit { } impl Waiter { - const QUEUED: usize = 0b1; + const UNQUEUED: usize = 1 << 16; + fn new() -> Self { + Waiter { + waker: AtomicWaker::new(), + state: AtomicUsize::new(Self::UNQUEUED), + } + } /// Assign permits to the waiter. /// @@ -390,7 +411,7 @@ impl Future for Acquire<'_> { permit.state = match permit.state { PermitState::Acquired(n) if n >= needed => return Ready(Ok(())), PermitState::Acquired(n) => { - ready!(semaphore.poll_acquire(cx, needed, node))?; + ready!(semaphore.poll_acquire(cx, needed - n, node))?; PermitState::Acquired(needed) } PermitState::Waiting(_n) => { From 8836c3896c95ba01dadc19da13a82ed140cf4572 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 27 Feb 2020 14:14:57 -0800 Subject: [PATCH 09/91] fix close behavior Signed-off-by: Eliza Weisman --- tokio/src/sync/semaphore_ll.rs | 35 +++--- tokio/src/sync/tests/semaphore_ll.rs | 157 ++++++++++++++------------- 2 files changed, 102 insertions(+), 90 deletions(-) diff --git a/tokio/src/sync/semaphore_ll.rs b/tokio/src/sync/semaphore_ll.rs index c0469a98d87..66bf3305782 100644 --- a/tokio/src/sync/semaphore_ll.rs +++ b/tokio/src/sync/semaphore_ll.rs @@ -61,16 +61,12 @@ struct Waiter { state: AtomicUsize, } -const CLOSED: usize = std::usize::MAX; +const CLOSED: usize = 1 << 17; impl Semaphore { /// Creates a new semaphore with the initial number of permits - /// - /// # Panics - /// - /// Panics if `permits` is zero. pub(crate) fn new(permits: usize) -> Self { - assert!(permits > 0, "number of permits must be greater than 0"); + assert!(permits <= std::u16::MAX as usize); Self { permits: AtomicUsize::new(permits), waiters: Mutex::new(LinkedList::new()), @@ -80,7 +76,7 @@ impl Semaphore { /// Returns the current number of available permits pub(crate) fn available_permits(&self) -> usize { - self.permits.load(Ordering::Acquire) + self.permits.load(Ordering::Acquire) & std::u16::MAX as usize } /// Closes the semaphore. This prevents the semaphore from issuing new @@ -88,6 +84,7 @@ impl Semaphore { pub(crate) fn close(&self) { // Acquire the `add_lock`, setting the "closed" flag on the lock. let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); + self.permits.fetch_or(CLOSED, Ordering::Release); if prev != 0 { // Another thread has the lock and will be responsible for notifying @@ -100,6 +97,7 @@ impl Semaphore { /// Adds `n` new permits to the semaphore. pub(crate) fn add_permits(&self, n: usize) { + dbg!(n); if n == 0 { return; } @@ -107,11 +105,18 @@ impl Semaphore { // TODO: Handle overflow. A panic is not sufficient, the process must // abort. let prev = self.add_lock.fetch_add(n << 1, Ordering::AcqRel); - if prev != 0 { + + let closed = match dbg!(prev) { + 1 => true, + 0 => false, // Another thread has the lock and will be responsible for notifying // pending waiters. - return; - } + _ => return, + }; + + // if self.permits.load(Ordering::Acquire) & CLOSED == 1 { + + // } self.add_permits_locked(n, false, self.waiters.lock().unwrap()); } @@ -127,7 +132,7 @@ impl Semaphore { let initial = rem; // Release the permits and notify - while dbg!(rem > 0) { + while dbg!(rem > 0) || dbg!(closed) { let pop = match waiters.last() { Some(last) => dbg!(last.assign_permits(&mut rem, closed)), None => { @@ -153,7 +158,7 @@ impl Semaphore { actual }; - rem = (actual - initial) >> 1; + rem = actual.saturating_sub(initial) >> 1; } } @@ -167,7 +172,7 @@ impl Semaphore { let needed = needed as usize; let (acquired, remaining) = loop { dbg!(needed, curr); - if curr == CLOSED { + if dbg!(curr & CLOSED == CLOSED) { return Ready(Err(AcquireError(()))); } let mut remaining = 0; @@ -353,6 +358,7 @@ impl Waiter { // Number of permits to assign to this waiter let assign = cmp::min(curr, *n); let next = curr - assign; + let next = if closed { next | CLOSED } else { next }; match self .state .compare_exchange(curr, next, Ordering::AcqRel, Ordering::Acquire) @@ -366,6 +372,9 @@ impl Waiter { self.waker.wake(); } + return true; + } else if closed { + self.waker.wake(); return true; } else { return false; diff --git a/tokio/src/sync/tests/semaphore_ll.rs b/tokio/src/sync/tests/semaphore_ll.rs index f8a2d9f1ce6..d7b2be161d8 100644 --- a/tokio/src/sync/tests/semaphore_ll.rs +++ b/tokio/src/sync/tests/semaphore_ll.rs @@ -175,25 +175,26 @@ fn poll_acquire_many_unavailable() { // Try to acquire the third permit let mut acquire_3 = task::spawn(permit_3.acquire(3, &s)); assert_pending!(acquire_3.poll()); - // permit_3.enter(|cx, mut p| {}); + assert_eq!(s.available_permits(), 0); - // permit_1.release(1, &s); + permit_1.release(1, &s); - // assert_eq!(s.available_permits(), 0); - // assert!(permit_2.is_woken()); - // assert_ready_ok!(permit_2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); + assert_eq!(s.available_permits(), 0); + assert!(acquire_2.is_woken()); + assert_ready_ok!(acquire_2.poll()); - // assert!(!permit_3.is_woken()); - // assert_eq!(s.available_permits(), 0); + assert!(!acquire_3.is_woken()); + assert_eq!(s.available_permits(), 0); - // permit_2.release(1, &s); - // assert!(!permit_3.is_woken()); - // assert_eq!(s.available_permits(), 0); + drop(acquire_2); // drop the acquire future so we can now + permit_2.release(1, &s); + assert!(!acquire_3.is_woken()); + assert_eq!(s.available_permits(), 0); - // permit_2.release(2, &s); - // assert!(permit_3.is_woken()); + permit_2.release(2, &s); + assert!(acquire_3.is_woken()); - // assert_ready_ok!(permit_3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); + assert_ready_ok!(acquire_3.poll()); } // #[test] @@ -243,97 +244,99 @@ fn poll_acquire_many_unavailable() { // assert_eq!(s.available_permits(), 2); // } -// #[test] -// fn poll_acquire_one_zero_permits() { -// let s = Semaphore::new(0); -// assert_eq!(s.available_permits(), 0); +#[test] +fn poll_acquire_one_zero_permits() { + let s = Semaphore::new(0); + assert_eq!(s.available_permits(), 0); -// let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = Permit::new(); -// // Try to acquire the permit -// permit.enter(|cx, mut p| { -// assert_pending!(p.as_mut().poll_acquire(cx, 1, &s)); -// }); + // Try to acquire the permit + let mut acquire = task::spawn(permit.acquire(1, &s)); + assert_pending!(acquire.poll()); -// s.add_permits(1); + s.add_permits(1); -// assert!(permit.is_woken()); -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -// } + assert!(acquire.is_woken()); + assert_ready_ok!(acquire.poll()); +} -// #[test] -// #[should_panic] -// fn validates_max_permits() { -// use std::usize; -// Semaphore::new((usize::MAX >> 2) + 1); -// } +#[test] +#[should_panic] +fn validates_max_permits() { + use std::usize; + Semaphore::new((usize::MAX >> 2) + 1); +} -// #[test] -// fn close_semaphore_prevents_acquire() { -// let s = Semaphore::new(5); -// s.close(); +#[test] +fn close_semaphore_prevents_acquire() { + let s = Semaphore::new(5); + s.close(); -// assert_eq!(5, s.available_permits()); + assert_eq!(5, s.available_permits()); -// let mut permit_1 = task::spawn(Box::new(Permit::new())); -// let mut permit_2 = task::spawn(Box::new(Permit::new())); + let mut permit_1 = Permit::new(); + let mut permit_2 = Permit::new(); -// assert_ready_err!(permit_1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -// assert_eq!(5, s.available_permits()); + assert_ready_err!(task::spawn(permit_1.acquire(1, &s)).poll()); + assert_eq!(5, s.available_permits()); -// assert_ready_err!(permit_2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// assert_eq!(5, s.available_permits()); -// } + assert_ready_err!(task::spawn(permit_2.acquire(1, &s)).poll()); + assert_eq!(5, s.available_permits()); +} -// #[test] -// fn close_semaphore_notifies_permit1() { -// let s = Semaphore::new(0); -// let mut permit = task::spawn(Box::new(Permit::new())); +#[test] +fn close_semaphore_notifies_permit1() { + let s = Semaphore::new(0); + let mut permit = Permit::new(); + let mut acquire = task::spawn(permit.acquire(1, &s)); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_pending!(acquire.poll()); -// s.close(); + s.close(); -// assert!(permit.is_woken()); -// assert_ready_err!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -// } + assert!(acquire.is_woken()); + assert_ready_err!(acquire.poll()); +} -// #[test] -// fn close_semaphore_notifies_permit2() { -// let s = Semaphore::new(2); +#[test] +fn close_semaphore_notifies_permit2() { + let s = Semaphore::new(2); -// let mut permit1 = task::spawn(Box::new(Permit::new())); -// let mut permit2 = task::spawn(Box::new(Permit::new())); -// let mut permit3 = task::spawn(Box::new(Permit::new())); -// let mut permit4 = task::spawn(Box::new(Permit::new())); + let mut permit1 = Permit::new(); + let mut permit2 = Permit::new(); + let mut permit3 = Permit::new(); + let mut permit4 = Permit::new(); -// // Acquire a couple of permits -// assert_ready_ok!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -// assert_ready_ok!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + // Acquire a couple of permits + assert_ready_ok!(task::spawn(permit1.acquire(1, &s)).poll()); + assert_ready_ok!(task::spawn(permit2.acquire(1, &s)).poll()); -// assert_pending!(permit3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -// assert_pending!(permit4.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + let mut acquire3 = task::spawn(permit3.acquire(1, &s)); + let mut acquire4 = task::spawn(permit4.acquire(1, &s)); + assert_pending!(acquire3.poll()); + assert_pending!(acquire4.poll()); -// s.close(); + s.close(); -// assert!(permit3.is_woken()); -// assert!(permit4.is_woken()); + assert!(acquire3.is_woken()); + assert!(acquire4.is_woken()); -// assert_ready_err!(permit3.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -// assert_ready_err!(permit4.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_ready_err!(acquire3.poll()); + assert_ready_err!(acquire4.poll()); -// assert_eq!(0, s.available_permits()); + assert_eq!(0, s.available_permits()); -// permit1.release(1, &s); + permit1.release(1, &s); -// assert_eq!(1, s.available_permits()); + assert_eq!(1, s.available_permits()); -// assert_ready_err!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_ready_err!(task::spawn(permit1.acquire(1, &s)).poll()); -// permit2.release(1, &s); + permit2.release(1, &s); -// assert_eq!(2, s.available_permits()); -// } + assert_eq!(2, s.available_permits()); +} // #[test] // fn poll_acquire_additional_permits_while_waiting_before_assigned() { From 5ee03b10e952c946ed69d25229d8e073cb06f340 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 27 Feb 2020 15:06:49 -0800 Subject: [PATCH 10/91] update to use new linked list Signed-off-by: Eliza Weisman --- tokio/src/sync/semaphore_ll.rs | 68 ++++++++++++++++++++++------------ tokio/src/util/linked_list.rs | 9 +++++ 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/tokio/src/sync/semaphore_ll.rs b/tokio/src/sync/semaphore_ll.rs index 66bf3305782..e3ccb0dcb15 100644 --- a/tokio/src/sync/semaphore_ll.rs +++ b/tokio/src/sync/semaphore_ll.rs @@ -1,5 +1,4 @@ use crate::loom::{ - alloc, future::AtomicWaker, sync::{atomic::AtomicUsize, Mutex, MutexGuard}, }; @@ -8,6 +7,7 @@ use crate::util::linked_list::{self, LinkedList}; use std::{ cmp, fmt, future::Future, + marker::PhantomPinned, pin::Pin, ptr::NonNull, sync::atomic::Ordering, @@ -39,7 +39,7 @@ pub(crate) struct Permit { } pub(crate) struct Acquire<'a> { - node: linked_list::Entry, + node: Waiter, semaphore: &'a Semaphore, permit: &'a mut Permit, num_permits: u16, @@ -59,6 +59,12 @@ enum PermitState { struct Waiter { waker: AtomicWaker, state: AtomicUsize, + + /// Intrusive linked-list pointers + pointers: linked_list::Pointers, + + /// Should not be `Unpin`. + _p: PhantomPinned, } const CLOSED: usize = 1 << 17; @@ -166,7 +172,7 @@ impl Semaphore { &self, cx: &mut Context<'_>, needed: u16, - node: Pin<&mut linked_list::Entry>, + node: Pin<&mut Waiter>, ) -> Poll> { let mut curr = self.permits.load(Ordering::Acquire); let needed = needed as usize; @@ -198,9 +204,7 @@ impl Semaphore { } assert!(node.is_unlinked()); - let waiter = unsafe { &*node.get() }; - waiter - .state + node.state .compare_exchange( Waiter::UNQUEUED, remaining, @@ -209,10 +213,12 @@ impl Semaphore { ) .expect("not unqueued"); // otherwise, register the waker & enqueue the node. - waiter.waker.register_by_ref(cx.waker()); + node.waker.register_by_ref(cx.waker()); let mut queue = self.waiters.lock().unwrap(); unsafe { + // XXX(eliza) T_T + let node = Pin::into_inner_unchecked(node) as *mut _; queue.push_front(node); println!("enqueue"); } @@ -267,7 +273,7 @@ impl Permit { PermitState::Acquired(n) => PermitState::Acquired(n), }; Acquire { - node: linked_list::Entry::new(Waiter::new()), + node: Waiter::new(), semaphore, permit: self, num_permits, @@ -345,9 +351,15 @@ impl Waiter { Waiter { waker: AtomicWaker::new(), state: AtomicUsize::new(Self::UNQUEUED), + pointers: linked_list::Pointers::new(), + _p: PhantomPinned, } } + fn is_unlinked(&self) -> bool { + self.pointers.is_unlinked() + } + /// Assign permits to the waiter. /// /// Returns `true` if the waiter should be removed from the queue @@ -425,7 +437,7 @@ impl Future for Acquire<'_> { } PermitState::Waiting(_n) => { assert_eq!(_n, needed, "how the heck did you get in this state?"); - if unsafe { &*node.get() }.state.load(Ordering::Acquire) > 0 { + if node.state.load(Ordering::Acquire) > 0 { ready!(semaphore.poll_acquire(cx, needed, node))?; } PermitState::Acquired(needed) @@ -436,14 +448,7 @@ impl Future for Acquire<'_> { } impl Acquire<'_> { - fn project( - self: Pin<&mut Self>, - ) -> ( - Pin<&mut linked_list::Entry>, - &Semaphore, - &mut Permit, - u16, - ) { + fn project(self: Pin<&mut Self>) -> (Pin<&mut Waiter>, &Semaphore, &mut Permit, u16) { fn is_unpin() {} unsafe { // Safety: all fields other than `node` are `Unpin` @@ -470,20 +475,16 @@ impl Drop for Acquire<'_> { // don't need to release permits return; } - - // Safety: if the node is linked, then we are already pinned - let (mut node, semaphore, permit, needed) = unsafe { Pin::new_unchecked(self).project() }; - // This is where we ensure safety. The future is being dropped, // which means we must ensure that the waiter entry is no longer stored // in the linked list. - let mut waiters = semaphore.waiters.lock().unwrap(); + let mut waiters = self.semaphore.waiters.lock().unwrap(); // remove the entry from the list // // safety: the waiter is only added to `waiters` by virtue of it // being the only `LinkedList` available to the type. - unsafe { waiters.remove(node.as_mut()) }; + unsafe { waiters.remove(NonNull::from(&mut self.node)) }; // TODO(eliza): release permits to next waiter } @@ -540,3 +541,24 @@ impl fmt::Display for TryAcquireError { } impl std::error::Error for TryAcquireError {} + +/// # Safety +/// +/// `Waiter` is forced to be !Unpin. +unsafe impl linked_list::Link for Waiter { + type Handle = *mut Waiter; + type Target = Waiter; + + fn to_raw(handle: *mut Waiter) -> NonNull { + debug_assert!(!handle.is_null()); + unsafe { NonNull::new_unchecked(handle) } + } + + unsafe fn from_raw(ptr: NonNull) -> *mut Waiter { + ptr.as_ptr() + } + + unsafe fn pointers(mut target: NonNull) -> NonNull> { + NonNull::from(&mut target.as_mut().pointers) + } +} diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 57540c4a45d..597b582acee 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -117,6 +117,10 @@ impl LinkedList { } } + pub(crate) fn last<'a>(&'a self) -> Option<&'a T::Target> { + unsafe { Some(&*self.tail?.as_ptr()) } + } + /// Returns whether the linked list doesn not contain any node pub(crate) fn is_empty(&self) -> bool { if self.head.is_some() { @@ -174,6 +178,11 @@ impl Pointers { next: None, } } + + /// Returns `true` if this set of pointers is not part of a list. + pub(crate) fn is_unlinked(&self) -> bool { + self.prev.is_none() && self.next.is_none() + } } #[cfg(test)] From c4c685b4ba34499067af85e4364d6c4301591e2f Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 28 Feb 2020 16:13:01 -0800 Subject: [PATCH 11/91] bring back old semaphore_ll so channels can work Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 566 ++++++++++ tokio/src/sync/mod.rs | 2 +- tokio/src/sync/mpsc/chan.rs | 29 +- tokio/src/sync/mutex.rs | 2 +- tokio/src/sync/rwlock.rs | 2 +- tokio/src/sync/semaphore.rs | 2 +- tokio/src/sync/semaphore_ll.rs | 1307 +++++++++++++++++------ tokio/src/sync/semaphore_ll_old.rs | 1219 --------------------- tokio/src/sync/tests/mod.rs | 1 + tokio/src/sync/tests/semaphore_batch.rs | 480 +++++++++ tokio/src/sync/tests/semaphore_ll.rs | 418 ++++---- 11 files changed, 2246 insertions(+), 1782 deletions(-) create mode 100644 tokio/src/sync/batch_semaphore.rs delete mode 100644 tokio/src/sync/semaphore_ll_old.rs create mode 100644 tokio/src/sync/tests/semaphore_batch.rs diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs new file mode 100644 index 00000000000..26488faf268 --- /dev/null +++ b/tokio/src/sync/batch_semaphore.rs @@ -0,0 +1,566 @@ +use crate::loom::{ + future::AtomicWaker, + sync::{atomic::AtomicUsize, Mutex, MutexGuard}, +}; +use crate::util::linked_list::{self, LinkedList}; + +use std::{ + cmp, fmt, + future::Future, + marker::PhantomPinned, + pin::Pin, + ptr::NonNull, + sync::atomic::Ordering, + task::{ + Context, Poll, + Poll::{Pending, Ready}, + }, +}; + +pub(crate) struct Semaphore { + waiters: Mutex>, + permits: AtomicUsize, + add_lock: AtomicUsize, +} + +/// Error returned by `Permit::try_acquire`. +#[derive(Debug)] +pub(crate) enum TryAcquireError { + Closed, + NoPermits, +} +/// Error returned by `Permit::poll_acquire`. +#[derive(Debug)] +pub(crate) struct AcquireError(()); + +#[derive(Debug)] +pub(crate) struct Permit { + state: PermitState, +} + +pub(crate) struct Acquire<'a> { + node: Waiter, + semaphore: &'a Semaphore, + permit: &'a mut Permit, + num_permits: u16, +} + +/// Permit state +#[derive(Debug, Copy, Clone)] +enum PermitState { + /// Currently waiting for permits to be made available and assigned to the + /// waiter. + Waiting(u16), + + /// The number of acquired permits + Acquired(u16), +} + +struct Waiter { + waker: AtomicWaker, + state: AtomicUsize, + + /// Intrusive linked-list pointers + pointers: linked_list::Pointers, + + /// Should not be `Unpin`. + _p: PhantomPinned, +} + +const CLOSED: usize = 1 << 17; + +impl Semaphore { + /// Creates a new semaphore with the initial number of permits + pub(crate) fn new(permits: usize) -> Self { + assert!(permits <= std::u16::MAX as usize); + Self { + permits: AtomicUsize::new(permits), + waiters: Mutex::new(LinkedList::new()), + add_lock: AtomicUsize::new(0), + } + } + + /// Returns the current number of available permits + pub(crate) fn available_permits(&self) -> usize { + self.permits.load(Ordering::Acquire) & std::u16::MAX as usize + } + + /// Closes the semaphore. This prevents the semaphore from issuing new + /// permits and notifies all pending waiters. + pub(crate) fn close(&self) { + // Acquire the `add_lock`, setting the "closed" flag on the lock. + let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); + self.permits.fetch_or(CLOSED, Ordering::Release); + + if prev != 0 { + // Another thread has the lock and will be responsible for notifying + // pending waiters. + return; + } + + self.add_permits_locked(0, true, self.waiters.lock().unwrap()); + } + + /// Adds `n` new permits to the semaphore. + pub(crate) fn add_permits(&self, n: usize) { + dbg!(n); + if n == 0 { + return; + } + + // TODO: Handle overflow. A panic is not sufficient, the process must + // abort. + let prev = self.add_lock.fetch_add(n << 1, Ordering::AcqRel); + + let closed = match dbg!(prev) { + 1 => true, + 0 => false, + // Another thread has the lock and will be responsible for notifying + // pending waiters. + _ => return, + }; + + // if self.permits.load(Ordering::Acquire) & CLOSED == 1 { + + // } + + self.add_permits_locked(n, false, self.waiters.lock().unwrap()); + } + + fn add_permits_locked( + &self, + mut rem: usize, + mut closed: bool, + mut waiters: MutexGuard<'_, LinkedList>, + ) { + while rem > 0 || closed { + // how many permits are we releasing on this pass? + + let initial = rem; + // Release the permits and notify + while dbg!(rem > 0) || dbg!(closed) { + let pop = match waiters.last() { + Some(last) => dbg!(last.assign_permits(&mut rem, closed)), + None => { + self.permits.fetch_add(rem, Ordering::Release); + break; + // false + } + }; + if pop { + waiters.pop_back().unwrap(); + } + } + + let n = (initial - rem) << 1; + + let actual = if closed { + let actual = self.add_lock.fetch_sub(n | 1, Ordering::AcqRel); + closed = false; + actual + } else { + let actual = self.add_lock.fetch_sub(n, Ordering::AcqRel); + closed = actual & 1 == 1; + actual + }; + + rem = actual.saturating_sub(initial) >> 1; + dbg!(rem); + } + } + + fn poll_acquire( + &self, + cx: &mut Context<'_>, + needed: u16, + node: Pin<&mut Waiter>, + ) -> Poll> { + let mut curr = self.permits.load(Ordering::Acquire); + let needed = needed as usize; + let (acquired, remaining) = loop { + dbg!(needed, curr); + if dbg!(curr & CLOSED == CLOSED) { + return Ready(Err(AcquireError(()))); + } + let mut remaining = 0; + let (next, acquired) = if dbg!(curr) >= dbg!(needed) { + (curr - needed, needed) + } else { + remaining = needed - curr; + (0, curr) + }; + match self.permits.compare_exchange_weak( + curr, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break (acquired, remaining), + Err(actual) => curr = actual, + } + }; + dbg!(acquired, remaining); + if remaining == 0 { + return Ready(Ok(())); + } + + assert!(node.is_unlinked()); + node.state + .compare_exchange( + Waiter::UNQUEUED, + remaining, + Ordering::Release, + Ordering::Relaxed, + ) + .expect("not unqueued"); + // otherwise, register the waker & enqueue the node. + node.waker.register_by_ref(cx.waker()); + + let mut queue = self.waiters.lock().unwrap(); + unsafe { + // XXX(eliza) T_T + let node = Pin::into_inner_unchecked(node) as *mut _; + queue.push_front(node); + println!("enqueue"); + } + + Pending + } +} + +impl Drop for Semaphore { + fn drop(&mut self) { + self.close(); + } +} + +impl fmt::Debug for Semaphore { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("Semaphore") + .field("permits", &self.add_lock.load(Ordering::Relaxed)) + .field("add_lock", &self.add_lock.load(Ordering::Relaxed)) + .finish() + } +} + +impl Permit { + /// Creates a new `Permit`. + /// + /// The permit begins in the "unacquired" state. + pub(crate) fn new() -> Permit { + use PermitState::Acquired; + + Permit { state: Acquired(0) } + } + + /// Returns `true` if the permit has been acquired + pub(crate) fn is_acquired(&self) -> bool { + match self.state { + PermitState::Acquired(num) if num > 0 => true, + _ => false, + } + } + + /// Returns a future that tries to acquire the permit. If no permits are available, the current task + /// is notified once a new permit becomes available. + pub(crate) fn acquire<'a>( + &'a mut self, + num_permits: u16, + semaphore: &'a Semaphore, + ) -> Acquire<'a> { + self.state = match self.state { + PermitState::Acquired(0) => PermitState::Waiting(num_permits), + PermitState::Waiting(n) => PermitState::Waiting(cmp::max(n, num_permits)), + PermitState::Acquired(n) => PermitState::Acquired(n), + }; + Acquire { + node: Waiter::new(), + semaphore, + permit: self, + num_permits, + } + } + + /// Tries to acquire the permit. + pub(crate) fn try_acquire( + &mut self, + num_permits: u16, + semaphore: &Semaphore, + ) -> Result<(), TryAcquireError> { + unimplemented!() + } + + /// Releases a permit back to the semaphore + pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { + let n = self.forget(n, semaphore); + semaphore.add_permits(n as usize); + } + + /// Forgets the permit **without** releasing it back to the semaphore. + /// + /// After calling `forget`, `poll_acquire` is able to acquire new permit + /// from the sempahore. + /// + /// Repeatedly calling `forget` without associated calls to `add_permit` + /// will result in the semaphore losing all permits. + /// + /// Will forget **at most** the number of acquired permits. This number is + /// returned. + pub(crate) fn forget(&mut self, n: u16, semaphore: &Semaphore) -> u16 { + use PermitState::*; + + match self.state { + Waiting(requested) => { + panic!( + "cannot forget permits while in wait queue; we are already borrowed mutably?" + ) + // let n = cmp::min(n, requested); + // let node = unsafe { &*self.node.get() }; + // // Decrement + // let acquired = node.try_dec_permits_to_acquire(n as usize) as u16; + + // if n == requested { + // self.state = Acquired(0); + // // // TODO: rm from wait list here! + // // semaphore + // } else if acquired == requested - n { + // self.state = Waiting(acquired); + // } else { + // self.state = Waiting(requested - n); + // } + + // acquired + } + Acquired(acquired) => { + let n = cmp::min(n, acquired); + self.state = Acquired(acquired - n); + dbg!(n) + } + } + } +} + +impl Default for Permit { + fn default() -> Self { + Self::new() + } +} + +impl Waiter { + const UNQUEUED: usize = 1 << 16; + fn new() -> Self { + Waiter { + waker: AtomicWaker::new(), + state: AtomicUsize::new(Self::UNQUEUED), + pointers: linked_list::Pointers::new(), + _p: PhantomPinned, + } + } + + fn is_unlinked(&self) -> bool { + self.pointers.is_unlinked() + } + + /// Assign permits to the waiter. + /// + /// Returns `true` if the waiter should be removed from the queue + fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { + if closed { + self.waker.wake(); + return true; + } + let mut curr = self.state.load(Ordering::Acquire); + + loop { + // Number of permits to assign to this waiter + let assign = cmp::min(curr, *n); + let next = curr - assign; + let next = if closed { next | CLOSED } else { next }; + match self + .state + .compare_exchange(curr, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => { + // Update `n` + *n -= assign; + + if next == 0 { + if curr > 0 { + self.waker.wake(); + } + + return true; + } else { + return false; + } + } + Err(actual) => curr = actual, + } + } + } + + /// Try to decrement the number of permits to acquire. This returns the + /// actual number of permits that were decremented. The delta betweeen `n` + /// and the return has been assigned to the permit and the caller must + /// assign these back to the semaphore. + fn try_dec_permits_to_acquire(&self, n: usize) -> usize { + let mut curr = self.state.load(Ordering::Acquire); + + loop { + // if !curr.is_queued() { + // assert_eq!(0, curr.permits_to_acquire()); + // } + + let delta = cmp::min(n, curr); + let rem = curr - delta; + + match self + .state + .compare_exchange(curr, rem, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return n - delta, + Err(actual) => curr = actual, + } + } + } +} + +impl Future for Acquire<'_> { + type Output = Result<(), AcquireError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let (node, semaphore, permit, mut needed) = self.project(); + dbg!(&semaphore, &permit, &needed); + permit.state = match permit.state { + PermitState::Acquired(n) if n >= needed => return Ready(Ok(())), + PermitState::Acquired(n) => { + ready!(semaphore.poll_acquire(cx, needed - n, node))?; + PermitState::Acquired(needed) + } + PermitState::Waiting(_n) => { + assert_eq!(_n, needed, "how the heck did you get in this state?"); + if node.state.load(Ordering::Acquire) > 0 { + ready!(semaphore.poll_acquire(cx, needed, node))?; + } + PermitState::Acquired(needed) + } + }; + Ready(Ok(())) + } +} + +impl Acquire<'_> { + fn project(self: Pin<&mut Self>) -> (Pin<&mut Waiter>, &Semaphore, &mut Permit, u16) { + fn is_unpin() {} + unsafe { + // Safety: all fields other than `node` are `Unpin` + + is_unpin::<&Semaphore>(); + is_unpin::<&mut Permit>(); + is_unpin::(); + + let this = self.get_unchecked_mut(); + ( + Pin::new_unchecked(&mut this.node), + &this.semaphore, + &mut this.permit, + this.num_permits, + ) + } + } +} + +impl Drop for Acquire<'_> { + fn drop(&mut self) { + dbg!("drop acquire"); + if dbg!(self.node.is_unlinked()) { + // don't need to release permits + return; + } + // This is where we ensure safety. The future is being dropped, + // which means we must ensure that the waiter entry is no longer stored + // in the linked list. + let mut waiters = self.semaphore.waiters.lock().unwrap(); + + // remove the entry from the list + // + // safety: the waiter is only added to `waiters` by virtue of it + // being the only `LinkedList` available to the type. + unsafe { waiters.remove(NonNull::from(&mut self.node)) }; + + // TODO(eliza): release permits to next waiter + } +} + +// ===== impl AcquireError ==== + +impl AcquireError { + fn closed() -> AcquireError { + AcquireError(()) + } +} + +fn to_try_acquire(_: AcquireError) -> TryAcquireError { + TryAcquireError::Closed +} + +impl fmt::Display for AcquireError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(fmt, "semaphore closed") + } +} + +impl std::error::Error for AcquireError {} + +// ===== impl TryAcquireError ===== + +impl TryAcquireError { + /// Returns `true` if the error was caused by a closed semaphore. + pub(crate) fn is_closed(&self) -> bool { + match self { + TryAcquireError::Closed => true, + _ => false, + } + } + + /// Returns `true` if the error was caused by calling `try_acquire` on a + /// semaphore with no available permits. + pub(crate) fn is_no_permits(&self) -> bool { + match self { + TryAcquireError::NoPermits => true, + _ => false, + } + } +} + +impl fmt::Display for TryAcquireError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TryAcquireError::Closed => write!(fmt, "{}", "semaphore closed"), + TryAcquireError::NoPermits => write!(fmt, "{}", "no permits available"), + } + } +} + +impl std::error::Error for TryAcquireError {} + +/// # Safety +/// +/// `Waiter` is forced to be !Unpin. +unsafe impl linked_list::Link for Waiter { + type Handle = *mut Waiter; + type Target = Waiter; + + fn to_raw(handle: *mut Waiter) -> NonNull { + debug_assert!(!handle.is_null()); + unsafe { NonNull::new_unchecked(handle) } + } + + unsafe fn from_raw(ptr: NonNull) -> *mut Waiter { + ptr.as_ptr() + } + + unsafe fn pointers(mut target: NonNull) -> NonNull> { + NonNull::from(&mut target.as_mut().pointers) + } +} diff --git a/tokio/src/sync/mod.rs b/tokio/src/sync/mod.rs index ea8fdff3afd..0607f78ad42 100644 --- a/tokio/src/sync/mod.rs +++ b/tokio/src/sync/mod.rs @@ -435,7 +435,7 @@ cfg_sync! { pub mod oneshot; - // pub(crate) mod semaphore_ll2; + pub(crate) mod batch_semaphore; pub(crate) mod semaphore_ll; mod semaphore; pub use semaphore::{Semaphore, SemaphorePermit}; diff --git a/tokio/src/sync/mpsc/chan.rs b/tokio/src/sync/mpsc/chan.rs index 4151cf787cd..2fc915d0e83 100644 --- a/tokio/src/sync/mpsc/chan.rs +++ b/tokio/src/sync/mpsc/chan.rs @@ -6,7 +6,6 @@ use crate::sync::mpsc::error::{ClosedError, TryRecvError}; use crate::sync::mpsc::{error, list}; use std::fmt; -use std::pin::Pin; use std::process; use std::sync::atomic::Ordering::{AcqRel, Relaxed}; use std::task::Poll::{Pending, Ready}; @@ -85,7 +84,7 @@ pub(crate) trait Semaphore { fn poll_acquire( &self, cx: &mut Context<'_>, - permit: Pin<&mut Self::Permit>, + permit: &mut Self::Permit, ) -> Poll>; fn try_acquire(&self, permit: &mut Self::Permit) -> Result<(), TrySendError>; @@ -187,15 +186,8 @@ where } } - pub(crate) fn poll_ready( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { - let (inner, permit) = unsafe { - let this = self.get_unchecked_mut(); - (&this.inner, Pin::new_unchecked(&mut this.permit)) - }; - inner.semaphore.poll_acquire(cx, permit) + pub(crate) fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.semaphore.poll_acquire(cx, &mut self.permit) } /// Send a message and notify the receiver. @@ -430,12 +422,11 @@ impl Semaphore for (crate::sync::semaphore_ll::Semaphore, usize) { fn poll_acquire( &self, cx: &mut Context<'_>, - permit: Pin<&mut Permit>, + permit: &mut Permit, ) -> Poll> { - // permit - // .poll_acquire(cx, 1, &self.0) - // .map_err(|_| ClosedError::new()) - unimplemented!() + permit + .poll_acquire(cx, 1, &self.0) + .map_err(|_| ClosedError::new()) } fn try_acquire(&self, permit: &mut Permit) -> Result<(), TrySendError> { @@ -444,7 +435,7 @@ impl Semaphore for (crate::sync::semaphore_ll::Semaphore, usize) { } fn forget(&self, permit: &mut Self::Permit) { - permit.forget(1, &self.0); + permit.forget(1); } fn close(&self) { @@ -480,9 +471,9 @@ impl Semaphore for AtomicUsize { fn poll_acquire( &self, _cx: &mut Context<'_>, - permit: Pin<&mut ()>, + permit: &mut (), ) -> Poll> { - Ready(self.try_acquire(&mut ()).map_err(|_| ClosedError::new())) + Ready(self.try_acquire(permit).map_err(|_| ClosedError::new())) } fn try_acquire(&self, _permit: &mut ()) -> Result<(), TrySendError> { diff --git a/tokio/src/sync/mutex.rs b/tokio/src/sync/mutex.rs index cd01ab32b8e..39f33402baa 100644 --- a/tokio/src/sync/mutex.rs +++ b/tokio/src/sync/mutex.rs @@ -80,7 +80,7 @@ //! [`MutexGuard`]: struct.MutexGuard.html use crate::future::poll_fn; -use crate::sync::semaphore_ll as semaphore; +use crate::sync::batch_semaphore as semaphore; use std::cell::UnsafeCell; use std::error::Error; diff --git a/tokio/src/sync/rwlock.rs b/tokio/src/sync/rwlock.rs index 39e06fecf14..5992ab47f16 100644 --- a/tokio/src/sync/rwlock.rs +++ b/tokio/src/sync/rwlock.rs @@ -1,5 +1,5 @@ use crate::future::poll_fn; -use crate::sync::semaphore_ll::{AcquireError, Permit, Semaphore}; +use crate::sync::batch_semaphore::{AcquireError, Permit, Semaphore}; use std::cell::UnsafeCell; use std::ops; use std::pin::Pin; diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index df53a3311c2..3216ce1cd9f 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -1,4 +1,4 @@ -use super::semaphore_ll as ll; // low level implementation +use super::batch_semaphore as ll; // low level implementation use crate::future::poll_fn; use std::pin::Pin; diff --git a/tokio/src/sync/semaphore_ll.rs b/tokio/src/sync/semaphore_ll.rs index e3ccb0dcb15..69fd4a6a5d3 100644 --- a/tokio/src/sync/semaphore_ll.rs +++ b/tokio/src/sync/semaphore_ll.rs @@ -1,50 +1,100 @@ -use crate::loom::{ - future::AtomicWaker, - sync::{atomic::AtomicUsize, Mutex, MutexGuard}, -}; -use crate::util::linked_list::{self, LinkedList}; - -use std::{ - cmp, fmt, - future::Future, - marker::PhantomPinned, - pin::Pin, - ptr::NonNull, - sync::atomic::Ordering, - task::{ - Context, Poll, - Poll::{Pending, Ready}, - }, -}; - +#![cfg_attr(not(feature = "sync"), allow(dead_code, unreachable_pub))] + +//! Thread-safe, asynchronous counting semaphore. +//! +//! A `Semaphore` instance holds a set of permits. Permits are used to +//! synchronize access to a shared resource. +//! +//! Before accessing the shared resource, callers acquire a permit from the +//! semaphore. Once the permit is acquired, the caller then enters the critical +//! section. If no permits are available, then acquiring the semaphore returns +//! `Pending`. The task is woken once a permit becomes available. + +use crate::loom::cell::CausalCell; +use crate::loom::future::AtomicWaker; +use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize}; +use crate::loom::thread; + +use std::cmp; +use std::fmt; +use std::ptr::{self, NonNull}; +use std::sync::atomic::Ordering::{self, AcqRel, Acquire, Relaxed, Release}; +use std::task::Poll::{Pending, Ready}; +use std::task::{Context, Poll}; +use std::usize; + +/// Futures-aware semaphore. pub(crate) struct Semaphore { - waiters: Mutex>, - permits: AtomicUsize, - add_lock: AtomicUsize, + /// Tracks both the waiter queue tail pointer and the number of remaining + /// permits. + state: AtomicUsize, + + /// waiter queue head pointer. + head: CausalCell>, + + /// Coordinates access to the queue head. + rx_lock: AtomicUsize, + + /// Stub waiter node used as part of the MPSC channel algorithm. + stub: Box, } -/// Error returned by `Permit::try_acquire`. +/// A semaphore permit +/// +/// Tracks the lifecycle of a semaphore permit. +/// +/// An instance of `Permit` is intended to be used with a **single** instance of +/// `Semaphore`. Using a single instance of `Permit` with multiple semaphore +/// instances will result in unexpected behavior. +/// +/// `Permit` does **not** release the permit back to the semaphore on drop. It +/// is the user's responsibility to ensure that `Permit::release` is called +/// before dropping the permit. #[derive(Debug)] -pub(crate) enum TryAcquireError { - Closed, - NoPermits, +pub(crate) struct Permit { + waiter: Option>, + state: PermitState, } + /// Error returned by `Permit::poll_acquire`. #[derive(Debug)] pub(crate) struct AcquireError(()); +/// Error returned by `Permit::try_acquire`. #[derive(Debug)] -pub(crate) struct Permit { - state: PermitState, +pub(crate) enum TryAcquireError { + Closed, + NoPermits, } -pub(crate) struct Acquire<'a> { - node: Waiter, - semaphore: &'a Semaphore, - permit: &'a mut Permit, - num_permits: u16, +/// Node used to notify the semaphore waiter when permit is available. +#[derive(Debug)] +struct Waiter { + /// Stores waiter state. + /// + /// See `WaiterState` for more details. + state: AtomicUsize, + + /// Task to wake when a permit is made available. + waker: AtomicWaker, + + /// Next pointer in the queue of waiting senders. + next: AtomicPtr, } +/// Semaphore state +/// +/// The 2 low bits track the modes. +/// +/// - Closed +/// - Full +/// +/// When not full, the rest of the `usize` tracks the total number of messages +/// in the channel. When full, the rest of the `usize` is a pointer to the tail +/// of the "waiting senders" queue. +#[derive(Copy, Clone)] +struct SemState(usize); + /// Permit state #[derive(Debug, Copy, Clone)] enum PermitState { @@ -56,41 +106,224 @@ enum PermitState { Acquired(u16), } -struct Waiter { - waker: AtomicWaker, - state: AtomicUsize, +/// State for an individual waker node +#[derive(Debug, Copy, Clone)] +struct WaiterState(usize); - /// Intrusive linked-list pointers - pointers: linked_list::Pointers, +/// Waiter node is in the semaphore queue +const QUEUED: usize = 0b001; - /// Should not be `Unpin`. - _p: PhantomPinned, -} +/// Semaphore has been closed, no more permits will be issued. +const CLOSED: usize = 0b10; + +/// The permit that owns the `Waiter` dropped. +const DROPPED: usize = 0b100; + +/// Represents "one requested permit" in the waiter state +const PERMIT_ONE: usize = 0b1000; -const CLOSED: usize = 1 << 17; +/// Masks the waiter state to only contain bits tracking number of requested +/// permits. +const PERMIT_MASK: usize = usize::MAX - (PERMIT_ONE - 1); + +/// How much to shift a permit count to pack it into the waker state +const PERMIT_SHIFT: u32 = PERMIT_ONE.trailing_zeros(); + +/// Flag differentiating between available permits and waiter pointers. +/// +/// If we assume pointers are properly aligned, then the least significant bit +/// will always be zero. So, we use that bit to track if the value represents a +/// number. +const NUM_FLAG: usize = 0b01; + +/// Signal the semaphore is closed +const CLOSED_FLAG: usize = 0b10; + +/// Maximum number of permits a semaphore can manage +const MAX_PERMITS: usize = usize::MAX >> NUM_SHIFT; + +/// When representing "numbers", the state has to be shifted this much (to get +/// rid of the flag bit). +const NUM_SHIFT: usize = 2; + +// ===== impl Semaphore ===== impl Semaphore { /// Creates a new semaphore with the initial number of permits - pub(crate) fn new(permits: usize) -> Self { - assert!(permits <= std::u16::MAX as usize); - Self { - permits: AtomicUsize::new(permits), - waiters: Mutex::new(LinkedList::new()), - add_lock: AtomicUsize::new(0), + /// + /// # Panics + /// + /// Panics if `permits` is zero. + pub(crate) fn new(permits: usize) -> Semaphore { + let stub = Box::new(Waiter::new()); + let ptr = NonNull::from(&*stub); + + // Allocations are aligned + debug_assert!(ptr.as_ptr() as usize & NUM_FLAG == 0); + + let state = SemState::new(permits, &stub); + + Semaphore { + state: AtomicUsize::new(state.to_usize()), + head: CausalCell::new(ptr), + rx_lock: AtomicUsize::new(0), + stub, } } /// Returns the current number of available permits pub(crate) fn available_permits(&self) -> usize { - self.permits.load(Ordering::Acquire) & std::u16::MAX as usize + let curr = SemState(self.state.load(Acquire)); + curr.available_permits() + } + + /// Tries to acquire the requested number of permits, registering the waiter + /// if not enough permits are available. + fn poll_acquire( + &self, + cx: &mut Context<'_>, + num_permits: u16, + permit: &mut Permit, + ) -> Poll> { + self.poll_acquire2(num_permits, || { + let waiter = permit.waiter.get_or_insert_with(|| Box::new(Waiter::new())); + + waiter.waker.register_by_ref(cx.waker()); + + Some(NonNull::from(&**waiter)) + }) + } + + fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { + match self.poll_acquire2(num_permits, || None) { + Poll::Ready(res) => res.map_err(to_try_acquire), + Poll::Pending => Err(TryAcquireError::NoPermits), + } + } + + /// Polls for a permit + /// + /// Tries to acquire available permits first. If unable to acquire a + /// sufficient number of permits, the caller's waiter is pushed onto the + /// semaphore's wait queue. + fn poll_acquire2( + &self, + num_permits: u16, + mut get_waiter: F, + ) -> Poll> + where + F: FnMut() -> Option>, + { + let num_permits = num_permits as usize; + + // Load the current state + let mut curr = SemState(self.state.load(Acquire)); + + // Saves a ref to the waiter node + let mut maybe_waiter: Option> = None; + + /// Used in branches where we attempt to push the waiter into the wait + /// queue but fail due to permits becoming available or the wait queue + /// transitioning to "closed". In this case, the waiter must be + /// transitioned back to the "idle" state. + macro_rules! revert_to_idle { + () => { + if let Some(waiter) = maybe_waiter { + unsafe { waiter.as_ref() }.revert_to_idle(); + } + }; + } + + loop { + let mut next = curr; + + if curr.is_closed() { + revert_to_idle!(); + return Ready(Err(AcquireError::closed())); + } + + let acquired = next.acquire_permits(num_permits, &self.stub); + + if !acquired { + // There are not enough available permits to satisfy the + // request. The permit transitions to a waiting state. + debug_assert!(curr.waiter().is_some() || curr.available_permits() < num_permits); + + if let Some(waiter) = maybe_waiter.as_ref() { + // Safety: the caller owns the waiter. + let w = unsafe { waiter.as_ref() }; + w.set_permits_to_acquire(num_permits - curr.available_permits()); + } else { + // Get the waiter for the permit. + if let Some(waiter) = get_waiter() { + // Safety: the caller owns the waiter. + let w = unsafe { waiter.as_ref() }; + + // If there are any currently available permits, the + // waiter acquires those immediately and waits for the + // remaining permits to become available. + if !w.to_queued(num_permits - curr.available_permits()) { + // The node is alrady queued, there is no further work + // to do. + return Pending; + } + + maybe_waiter = Some(waiter); + } else { + // No waiter, this indicates the caller does not wish to + // "wait", so there is nothing left to do. + return Pending; + } + } + + next.set_waiter(maybe_waiter.unwrap()); + } + + debug_assert_ne!(curr.0, 0); + debug_assert_ne!(next.0, 0); + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => { + if acquired { + // Successfully acquire permits **without** queuing the + // waiter node. The waiter node is not currently in the + // queue. + revert_to_idle!(); + return Ready(Ok(())); + } else { + // The node is pushed into the queue, the final step is + // to set the node's "next" pointer to return the wait + // queue into a consistent state. + + let prev_waiter = + curr.waiter().unwrap_or_else(|| NonNull::from(&*self.stub)); + + let waiter = maybe_waiter.unwrap(); + + // Link the nodes. + // + // Safety: the mpsc algorithm guarantees the old tail of + // the queue is not removed from the queue during the + // push process. + unsafe { + prev_waiter.as_ref().store_next(waiter); + } + + return Pending; + } + } + Err(actual) => { + curr = SemState(actual); + } + } + } } /// Closes the semaphore. This prevents the semaphore from issuing new /// permits and notifies all pending waiters. pub(crate) fn close(&self) { - // Acquire the `add_lock`, setting the "closed" flag on the lock. - let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); - self.permits.fetch_or(CLOSED, Ordering::Release); + // Acquire the `rx_lock`, setting the "closed" flag on the lock. + let prev = self.rx_lock.fetch_or(1, AcqRel); if prev != 0 { // Another thread has the lock and will be responsible for notifying @@ -98,132 +331,246 @@ impl Semaphore { return; } - self.add_permits_locked(0, true, self.waiters.lock().unwrap()); + self.add_permits_locked(0, true); } /// Adds `n` new permits to the semaphore. pub(crate) fn add_permits(&self, n: usize) { - dbg!(n); if n == 0 { return; } // TODO: Handle overflow. A panic is not sufficient, the process must // abort. - let prev = self.add_lock.fetch_add(n << 1, Ordering::AcqRel); + let prev = self.rx_lock.fetch_add(n << 1, AcqRel); - let closed = match dbg!(prev) { - 1 => true, - 0 => false, + if prev != 0 { // Another thread has the lock and will be responsible for notifying // pending waiters. - _ => return, - }; - - // if self.permits.load(Ordering::Acquire) & CLOSED == 1 { - - // } + return; + } - self.add_permits_locked(n, false, self.waiters.lock().unwrap()); + self.add_permits_locked(n, false); } - fn add_permits_locked( - &self, - mut rem: usize, - mut closed: bool, - mut waiters: MutexGuard<'_, LinkedList>, - ) { + fn add_permits_locked(&self, mut rem: usize, mut closed: bool) { while rem > 0 || closed { - // how many permits are we releasing on this pass? + if closed { + SemState::fetch_set_closed(&self.state, AcqRel); + } - let initial = rem; // Release the permits and notify - while dbg!(rem > 0) || dbg!(closed) { - let pop = match waiters.last() { - Some(last) => dbg!(last.assign_permits(&mut rem, closed)), - None => { - self.permits.fetch_add(rem, Ordering::Release); - break; - // false - } - }; - if pop { - waiters.pop_back().unwrap(); - } - } + self.add_permits_locked2(rem, closed); - let n = (initial - rem) << 1; + let n = rem << 1; let actual = if closed { - let actual = self.add_lock.fetch_sub(n | 1, Ordering::AcqRel); + let actual = self.rx_lock.fetch_sub(n | 1, AcqRel); closed = false; actual } else { - let actual = self.add_lock.fetch_sub(n, Ordering::AcqRel); + let actual = self.rx_lock.fetch_sub(n, AcqRel); closed = actual & 1 == 1; actual }; - rem = actual.saturating_sub(initial) >> 1; + rem = (actual >> 1) - rem; } } - fn poll_acquire( - &self, - cx: &mut Context<'_>, - needed: u16, - node: Pin<&mut Waiter>, - ) -> Poll> { - let mut curr = self.permits.load(Ordering::Acquire); - let needed = needed as usize; - let (acquired, remaining) = loop { - dbg!(needed, curr); - if dbg!(curr & CLOSED == CLOSED) { - return Ready(Err(AcquireError(()))); + /// Releases a specific amount of permits to the semaphore + /// + /// This function is called by `add_permits` after the add lock has been + /// acquired. + fn add_permits_locked2(&self, mut n: usize, closed: bool) { + // If closing the semaphore, we want to drain the entire queue. The + // number of permits being assigned doesn't matter. + if closed { + n = usize::MAX; + } + + 'outer: while n > 0 { + unsafe { + let mut head = self.head.with(|head| *head); + let mut next_ptr = head.as_ref().next.load(Acquire); + + let stub = self.stub(); + + if head == stub { + // The stub node indicates an empty queue. Any remaining + // permits get assigned back to the semaphore. + let next = match NonNull::new(next_ptr) { + Some(next) => next, + None => { + // This loop is not part of the standard intrusive mpsc + // channel algorithm. This is where we atomically pop + // the last task and add `n` to the remaining capacity. + // + // This modification to the pop algorithm works because, + // at this point, we have not done any work (only done + // reading). We have a *pretty* good idea that there is + // no concurrent pusher. + // + // The capacity is then atomically added by doing an + // AcqRel CAS on `state`. The `state` cell is the + // linchpin of the algorithm. + // + // By successfully CASing `head` w/ AcqRel, we ensure + // that, if any thread was racing and entered a push, we + // see that and abort pop, retrying as it is + // "inconsistent". + let mut curr = SemState::load(&self.state, Acquire); + + loop { + if curr.has_waiter(&self.stub) { + // A waiter is being added concurrently. + // This is the MPSC queue's "inconsistent" + // state and we must loop and try again. + thread::yield_now(); + continue 'outer; + } + + // If closing, nothing more to do. + if closed { + debug_assert!(curr.is_closed(), "state = {:?}", curr); + return; + } + + let mut next = curr; + next.release_permits(n, &self.stub); + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => return, + Err(actual) => { + curr = SemState(actual); + } + } + } + } + }; + + self.head.with_mut(|head| *head = next); + head = next; + next_ptr = next.as_ref().next.load(Acquire); + } + + // `head` points to a waiter assign permits to the waiter. If + // all requested permits are satisfied, then we can continue, + // otherwise the node stays in the wait queue. + if !head.as_ref().assign_permits(&mut n, closed) { + assert_eq!(n, 0); + return; + } + + if let Some(next) = NonNull::new(next_ptr) { + self.head.with_mut(|head| *head = next); + + self.remove_queued(head, closed); + continue 'outer; + } + + let state = SemState::load(&self.state, Acquire); + + // This must always be a pointer as the wait list is not empty. + let tail = state.waiter().unwrap(); + + if tail != head { + // Inconsistent + thread::yield_now(); + continue 'outer; + } + + self.push_stub(closed); + + next_ptr = head.as_ref().next.load(Acquire); + + if let Some(next) = NonNull::new(next_ptr) { + self.head.with_mut(|head| *head = next); + + self.remove_queued(head, closed); + continue 'outer; + } + + // Inconsistent state, loop + thread::yield_now(); } - let mut remaining = 0; - let (next, acquired) = if dbg!(curr) >= dbg!(needed) { - (curr - needed, needed) - } else { - remaining = needed - curr; - (0, curr) - }; - match self.permits.compare_exchange_weak( - curr, - next, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => break (acquired, remaining), - Err(actual) => curr = actual, + } + } + + /// The wait node has had all of its permits assigned and has been removed + /// from the wait queue. + /// + /// Attempt to remove the QUEUED bit from the node. If additional permits + /// are concurrently requested, the node must be pushed back into the wait + /// queued. + fn remove_queued(&self, waiter: NonNull, closed: bool) { + let mut curr = WaiterState(unsafe { waiter.as_ref() }.state.load(Acquire)); + + loop { + if curr.is_dropped() { + // The Permit dropped, it is on us to release the memory + let _ = unsafe { Box::from_raw(waiter.as_ptr()) }; + return; + } + + // The node is removed from the queue. We attempt to unset the + // queued bit, but concurrently the waiter has requested more + // permits. When the waiter requested more permits, it saw the + // queued bit set so took no further action. This requires us to + // push the node back into the queue. + if curr.permits_to_acquire() > 0 { + // More permits are requested. The waiter must be re-queued + unsafe { + self.push_waiter(waiter, closed); + } + return; + } + + let mut next = curr; + next.unset_queued(); + + let w = unsafe { waiter.as_ref() }; + + match w.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => return, + Err(actual) => { + curr = WaiterState(actual); + } } - }; - dbg!(acquired, remaining); - if remaining == 0 { - return Ready(Ok(())); - } - - assert!(node.is_unlinked()); - node.state - .compare_exchange( - Waiter::UNQUEUED, - remaining, - Ordering::Release, - Ordering::Relaxed, - ) - .expect("not unqueued"); - // otherwise, register the waker & enqueue the node. - node.waker.register_by_ref(cx.waker()); - - let mut queue = self.waiters.lock().unwrap(); - unsafe { - // XXX(eliza) T_T - let node = Pin::into_inner_unchecked(node) as *mut _; - queue.push_front(node); - println!("enqueue"); - } - - Pending + } + } + + unsafe fn push_stub(&self, closed: bool) { + self.push_waiter(self.stub(), closed); + } + + unsafe fn push_waiter(&self, waiter: NonNull, closed: bool) { + // Set the next pointer. This does not require an atomic operation as + // this node is not accessible. The write will be flushed with the next + // operation + waiter.as_ref().next.store(ptr::null_mut(), Relaxed); + + // Update the tail to point to the new node. We need to see the previous + // node in order to update the next pointer as well as release `task` + // to any other threads calling `push`. + let next = SemState::new_ptr(waiter, closed); + let prev = SemState(self.state.swap(next.0, AcqRel)); + + debug_assert_eq!(closed, prev.is_closed()); + + // This function is only called when there are pending tasks. Because of + // this, the state must *always* be in pointer mode. + let prev = prev.waiter().unwrap(); + + // No cycles plz + debug_assert_ne!(prev, waiter); + + // Release `task` to the consume end. + prev.as_ref().next.store(waiter.as_ptr(), Release); + } + + fn stub(&self) -> NonNull { + unsafe { NonNull::new_unchecked(&*self.stub as *const _ as *mut _) } } } @@ -236,12 +583,19 @@ impl Drop for Semaphore { impl fmt::Debug for Semaphore { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_struct("Semaphore") - .field("permits", &self.add_lock.load(Ordering::Relaxed)) - .field("add_lock", &self.add_lock.load(Ordering::Relaxed)) + .field("state", &SemState::load(&self.state, Relaxed)) + .field("head", &self.head.with(|ptr| ptr)) + .field("rx_lock", &self.rx_lock.load(Relaxed)) + .field("stub", &self.stub) .finish() } } +unsafe impl Send for Semaphore {} +unsafe impl Sync for Semaphore {} + +// ===== impl Permit ===== + impl Permit { /// Creates a new `Permit`. /// @@ -249,7 +603,10 @@ impl Permit { pub(crate) fn new() -> Permit { use PermitState::Acquired; - Permit { state: Acquired(0) } + Permit { + waiter: None, + state: Acquired(0), + } } /// Returns `true` if the permit has been acquired @@ -260,23 +617,79 @@ impl Permit { } } - /// Returns a future that tries to acquire the permit. If no permits are available, the current task + /// Tries to acquire the permit. If no permits are available, the current task /// is notified once a new permit becomes available. - pub(crate) fn acquire<'a>( - &'a mut self, + pub(crate) fn poll_acquire( + &mut self, + cx: &mut Context<'_>, num_permits: u16, - semaphore: &'a Semaphore, - ) -> Acquire<'a> { - self.state = match self.state { - PermitState::Acquired(0) => PermitState::Waiting(num_permits), - PermitState::Waiting(n) => PermitState::Waiting(cmp::max(n, num_permits)), - PermitState::Acquired(n) => PermitState::Acquired(n), - }; - Acquire { - node: Waiter::new(), - semaphore, - permit: self, - num_permits, + semaphore: &Semaphore, + ) -> Poll> { + use std::cmp::Ordering::*; + use PermitState::*; + + match self.state { + Waiting(requested) => { + // There must be a waiter + let waiter = self.waiter.as_ref().unwrap(); + + match requested.cmp(&num_permits) { + Less => { + let delta = num_permits - requested; + + // Request additional permits. If the waiter has been + // dequeued, it must be re-queued. + if !waiter.try_inc_permits_to_acquire(delta as usize) { + let waiter = NonNull::from(&**waiter); + + // Ignore the result. The check for + // `permits_to_acquire()` will converge the state as + // needed + let _ = semaphore.poll_acquire2(delta, || Some(waiter))?; + } + + self.state = Waiting(num_permits); + } + Greater => { + let delta = requested - num_permits; + let to_release = waiter.try_dec_permits_to_acquire(delta as usize); + + semaphore.add_permits(to_release); + self.state = Waiting(num_permits); + } + Equal => {} + } + + if waiter.permits_to_acquire()? == 0 { + self.state = Acquired(requested); + return Ready(Ok(())); + } + + waiter.waker.register_by_ref(cx.waker()); + + if waiter.permits_to_acquire()? == 0 { + self.state = Acquired(requested); + return Ready(Ok(())); + } + + Pending + } + Acquired(acquired) => { + if acquired >= num_permits { + Ready(Ok(())) + } else { + match semaphore.poll_acquire(cx, num_permits - acquired, self)? { + Ready(()) => { + self.state = Acquired(num_permits); + Ready(Ok(())) + } + Pending => { + self.state = Waiting(num_permits); + Pending + } + } + } + } } } @@ -286,12 +699,49 @@ impl Permit { num_permits: u16, semaphore: &Semaphore, ) -> Result<(), TryAcquireError> { - unimplemented!() + use PermitState::*; + + match self.state { + Waiting(requested) => { + // There must be a waiter + let waiter = self.waiter.as_ref().unwrap(); + + if requested > num_permits { + let delta = requested - num_permits; + let to_release = waiter.try_dec_permits_to_acquire(delta as usize); + + semaphore.add_permits(to_release); + self.state = Waiting(num_permits); + } + + let res = waiter.permits_to_acquire().map_err(to_try_acquire)?; + + if res == 0 { + if requested < num_permits { + // Try to acquire the additional permits + semaphore.try_acquire(num_permits - requested)?; + } + + self.state = Acquired(num_permits); + Ok(()) + } else { + Err(TryAcquireError::NoPermits) + } + } + Acquired(acquired) => { + if acquired < num_permits { + semaphore.try_acquire(num_permits - acquired)?; + self.state = Acquired(num_permits); + } + + Ok(()) + } + } } /// Releases a permit back to the semaphore pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { - let n = self.forget(n, semaphore); + let n = self.forget(n); semaphore.add_permits(n as usize); } @@ -305,35 +755,34 @@ impl Permit { /// /// Will forget **at most** the number of acquired permits. This number is /// returned. - pub(crate) fn forget(&mut self, n: u16, semaphore: &Semaphore) -> u16 { + pub(crate) fn forget(&mut self, n: u16) -> u16 { use PermitState::*; match self.state { Waiting(requested) => { - panic!( - "cannot forget permits while in wait queue; we are already borrowed mutably?" - ) - // let n = cmp::min(n, requested); - // let node = unsafe { &*self.node.get() }; - // // Decrement - // let acquired = node.try_dec_permits_to_acquire(n as usize) as u16; - - // if n == requested { - // self.state = Acquired(0); - // // // TODO: rm from wait list here! - // // semaphore - // } else if acquired == requested - n { - // self.state = Waiting(acquired); - // } else { - // self.state = Waiting(requested - n); - // } - - // acquired + let n = cmp::min(n, requested); + + // Decrement + let acquired = self + .waiter + .as_ref() + .unwrap() + .try_dec_permits_to_acquire(n as usize) as u16; + + if n == requested { + self.state = Acquired(0); + } else if acquired == requested - n { + self.state = Waiting(acquired); + } else { + self.state = Waiting(requested - n); + } + + acquired } Acquired(acquired) => { let n = cmp::min(n, acquired); self.state = Acquired(acquired - n); - dbg!(n) + n } } } @@ -345,220 +794,426 @@ impl Default for Permit { } } +impl Drop for Permit { + fn drop(&mut self) { + if let Some(waiter) = self.waiter.take() { + // Set the dropped flag + let state = WaiterState(waiter.state.fetch_or(DROPPED, AcqRel)); + + if state.is_queued() { + // The waiter is stored in the queue. The semaphore will drop it + std::mem::forget(waiter); + } + } + } +} + +// ===== impl AcquireError ==== + +impl AcquireError { + fn closed() -> AcquireError { + AcquireError(()) + } +} + +fn to_try_acquire(_: AcquireError) -> TryAcquireError { + TryAcquireError::Closed +} + +impl fmt::Display for AcquireError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(fmt, "semaphore closed") + } +} + +impl std::error::Error for AcquireError {} + +// ===== impl TryAcquireError ===== + +impl TryAcquireError { + /// Returns `true` if the error was caused by a closed semaphore. + pub(crate) fn is_closed(&self) -> bool { + match self { + TryAcquireError::Closed => true, + _ => false, + } + } + + /// Returns `true` if the error was caused by calling `try_acquire` on a + /// semaphore with no available permits. + pub(crate) fn is_no_permits(&self) -> bool { + match self { + TryAcquireError::NoPermits => true, + _ => false, + } + } +} + +impl fmt::Display for TryAcquireError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TryAcquireError::Closed => write!(fmt, "{}", "semaphore closed"), + TryAcquireError::NoPermits => write!(fmt, "{}", "no permits available"), + } + } +} + +impl std::error::Error for TryAcquireError {} + +// ===== impl Waiter ===== + impl Waiter { - const UNQUEUED: usize = 1 << 16; - fn new() -> Self { + fn new() -> Waiter { Waiter { + state: AtomicUsize::new(0), waker: AtomicWaker::new(), - state: AtomicUsize::new(Self::UNQUEUED), - pointers: linked_list::Pointers::new(), - _p: PhantomPinned, + next: AtomicPtr::new(ptr::null_mut()), + } + } + + fn permits_to_acquire(&self) -> Result { + let state = WaiterState(self.state.load(Acquire)); + + if state.is_closed() { + Err(AcquireError(())) + } else { + Ok(state.permits_to_acquire()) + } + } + + /// Only increments the number of permits *if* the waiter is currently + /// queued. + /// + /// # Returns + /// + /// `true` if the number of permits to acquire has been incremented. `false` + /// otherwise. On `false`, the caller should use `Semaphore::poll_acquire`. + fn try_inc_permits_to_acquire(&self, n: usize) -> bool { + let mut curr = WaiterState(self.state.load(Acquire)); + + loop { + if !curr.is_queued() { + assert_eq!(0, curr.permits_to_acquire()); + return false; + } + + let mut next = curr; + next.set_permits_to_acquire(n + curr.permits_to_acquire()); + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => return true, + Err(actual) => curr = WaiterState(actual), + } + } + } + + /// Try to decrement the number of permits to acquire. This returns the + /// actual number of permits that were decremented. The delta betweeen `n` + /// and the return has been assigned to the permit and the caller must + /// assign these back to the semaphore. + fn try_dec_permits_to_acquire(&self, n: usize) -> usize { + let mut curr = WaiterState(self.state.load(Acquire)); + + loop { + if !curr.is_queued() { + assert_eq!(0, curr.permits_to_acquire()); + } + + let delta = cmp::min(n, curr.permits_to_acquire()); + let rem = curr.permits_to_acquire() - delta; + + let mut next = curr; + next.set_permits_to_acquire(rem); + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => return n - delta, + Err(actual) => curr = WaiterState(actual), + } + } + } + + /// Store the number of remaining permits needed to satisfy the waiter and + /// transition to the "QUEUED" state. + /// + /// # Returns + /// + /// `true` if the `QUEUED` bit was set as part of the transition. + fn to_queued(&self, num_permits: usize) -> bool { + let mut curr = WaiterState(self.state.load(Acquire)); + + // The waiter should **not** be waiting for any permits. + debug_assert_eq!(curr.permits_to_acquire(), 0); + + loop { + let mut next = curr; + next.set_permits_to_acquire(num_permits); + next.set_queued(); + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { + Ok(_) => { + if curr.is_queued() { + return false; + } else { + // Make sure the next pointer is null + self.next.store(ptr::null_mut(), Relaxed); + return true; + } + } + Err(actual) => curr = WaiterState(actual), + } } } - fn is_unlinked(&self) -> bool { - self.pointers.is_unlinked() + /// Set the number of permits to acquire. + /// + /// This function is only called when the waiter is being inserted into the + /// wait queue. Because of this, there are no concurrent threads that can + /// modify the state and using `store` is safe. + fn set_permits_to_acquire(&self, num_permits: usize) { + debug_assert!(WaiterState(self.state.load(Acquire)).is_queued()); + + let mut state = WaiterState(QUEUED); + state.set_permits_to_acquire(num_permits); + + self.state.store(state.0, Release); } /// Assign permits to the waiter. /// /// Returns `true` if the waiter should be removed from the queue fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { - let mut curr = self.state.load(Ordering::Acquire); + let mut curr = WaiterState(self.state.load(Acquire)); loop { + let mut next = curr; + // Number of permits to assign to this waiter - let assign = cmp::min(curr, *n); - let next = curr - assign; - let next = if closed { next | CLOSED } else { next }; - match self - .state - .compare_exchange(curr, next, Ordering::AcqRel, Ordering::Acquire) - { + let assign = cmp::min(curr.permits_to_acquire(), *n); + + // Assign the permits + next.set_permits_to_acquire(curr.permits_to_acquire() - assign); + + if closed { + next.set_closed(); + } + + match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { Ok(_) => { // Update `n` *n -= assign; - if next == 0 { - if curr > 0 { + if next.permits_to_acquire() == 0 { + if curr.permits_to_acquire() > 0 { self.waker.wake(); } - return true; - } else if closed { - self.waker.wake(); return true; } else { return false; } } - Err(actual) => curr = actual, + Err(actual) => curr = WaiterState(actual), } } } - /// Try to decrement the number of permits to acquire. This returns the - /// actual number of permits that were decremented. The delta betweeen `n` - /// and the return has been assigned to the permit and the caller must - /// assign these back to the semaphore. - fn try_dec_permits_to_acquire(&self, n: usize) -> usize { - let mut curr = self.state.load(Ordering::Acquire); + fn revert_to_idle(&self) { + // An idle node is not waiting on any permits + self.state.store(0, Relaxed); + } - loop { - // if !curr.is_queued() { - // assert_eq!(0, curr.permits_to_acquire()); - // } + fn store_next(&self, next: NonNull) { + self.next.store(next.as_ptr(), Release); + } +} - let delta = cmp::min(n, curr); - let rem = curr - delta; +// ===== impl SemState ===== - match self - .state - .compare_exchange(curr, rem, Ordering::AcqRel, Ordering::Acquire) - { - Ok(_) => return n - delta, - Err(actual) => curr = actual, - } +impl SemState { + /// Returns a new default `State` value. + fn new(permits: usize, stub: &Waiter) -> SemState { + assert!(permits <= MAX_PERMITS); + + if permits > 0 { + SemState((permits << NUM_SHIFT) | NUM_FLAG) + } else { + SemState(stub as *const _ as usize) } } -} -impl Future for Acquire<'_> { - type Output = Result<(), AcquireError>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let (node, semaphore, permit, mut needed) = self.project(); - dbg!(&semaphore, &permit, &needed); - permit.state = match permit.state { - PermitState::Acquired(n) if n >= needed => return Ready(Ok(())), - PermitState::Acquired(n) => { - ready!(semaphore.poll_acquire(cx, needed - n, node))?; - PermitState::Acquired(needed) - } - PermitState::Waiting(_n) => { - assert_eq!(_n, needed, "how the heck did you get in this state?"); - if node.state.load(Ordering::Acquire) > 0 { - ready!(semaphore.poll_acquire(cx, needed, node))?; - } - PermitState::Acquired(needed) - } - }; - Ready(Ok(())) + /// Returns a `State` tracking `ptr` as the tail of the queue. + fn new_ptr(tail: NonNull, closed: bool) -> SemState { + let mut val = tail.as_ptr() as usize; + + if closed { + val |= CLOSED_FLAG; + } + + SemState(val) + } + + /// Returns the amount of remaining capacity + fn available_permits(self) -> usize { + if !self.has_available_permits() { + return 0; + } + + self.0 >> NUM_SHIFT + } + + /// Returns `true` if the state has permits that can be claimed by a waiter. + fn has_available_permits(self) -> bool { + self.0 & NUM_FLAG == NUM_FLAG + } + + fn has_waiter(self, stub: &Waiter) -> bool { + !self.has_available_permits() && !self.is_stub(stub) } -} -impl Acquire<'_> { - fn project(self: Pin<&mut Self>) -> (Pin<&mut Waiter>, &Semaphore, &mut Permit, u16) { - fn is_unpin() {} - unsafe { - // Safety: all fields other than `node` are `Unpin` + /// Tries to atomically acquire specified number of permits. + /// + /// # Return + /// + /// Returns `true` if the specified number of permits were acquired, `false` + /// otherwise. Returning false does not mean that there are no more + /// available permits. + fn acquire_permits(&mut self, num: usize, stub: &Waiter) -> bool { + debug_assert!(num > 0); + + if self.available_permits() < num { + return false; + } + + debug_assert!(self.waiter().is_none()); - is_unpin::<&Semaphore>(); - is_unpin::<&mut Permit>(); - is_unpin::(); + self.0 -= num << NUM_SHIFT; - let this = self.get_unchecked_mut(); - ( - Pin::new_unchecked(&mut this.node), - &this.semaphore, - &mut this.permit, - this.num_permits, - ) + if self.0 == NUM_FLAG { + // Set the state to the stub pointer. + self.0 = stub as *const _ as usize; } + + true } -} -impl Drop for Acquire<'_> { - fn drop(&mut self) { - dbg!("drop acquire"); - if dbg!(self.node.is_unlinked()) { - // don't need to release permits + /// Releases permits + /// + /// Returns `true` if the permits were accepted. + fn release_permits(&mut self, permits: usize, stub: &Waiter) { + debug_assert!(permits > 0); + + if self.is_stub(stub) { + self.0 = (permits << NUM_SHIFT) | NUM_FLAG | (self.0 & CLOSED_FLAG); return; } - // This is where we ensure safety. The future is being dropped, - // which means we must ensure that the waiter entry is no longer stored - // in the linked list. - let mut waiters = self.semaphore.waiters.lock().unwrap(); - // remove the entry from the list - // - // safety: the waiter is only added to `waiters` by virtue of it - // being the only `LinkedList` available to the type. - unsafe { waiters.remove(NonNull::from(&mut self.node)) }; + debug_assert!(self.has_available_permits()); - // TODO(eliza): release permits to next waiter + self.0 += permits << NUM_SHIFT; } -} -// ===== impl AcquireError ==== + fn is_waiter(self) -> bool { + self.0 & NUM_FLAG == 0 + } -impl AcquireError { - fn closed() -> AcquireError { - AcquireError(()) + /// Returns the waiter, if one is set. + fn waiter(self) -> Option> { + if self.is_waiter() { + let waiter = NonNull::new(self.as_ptr()).expect("null pointer stored"); + + Some(waiter) + } else { + None + } } -} -fn to_try_acquire(_: AcquireError) -> TryAcquireError { - TryAcquireError::Closed -} + /// Assumes `self` represents a pointer + fn as_ptr(self) -> *mut Waiter { + (self.0 & !CLOSED_FLAG) as *mut Waiter + } -impl fmt::Display for AcquireError { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(fmt, "semaphore closed") + /// Sets to a pointer to a waiter. + /// + /// This can only be done from the full state. + fn set_waiter(&mut self, waiter: NonNull) { + let waiter = waiter.as_ptr() as usize; + debug_assert!(!self.is_closed()); + + self.0 = waiter; } -} -impl std::error::Error for AcquireError {} + fn is_stub(self, stub: &Waiter) -> bool { + self.as_ptr() as usize == stub as *const _ as usize + } -// ===== impl TryAcquireError ===== + /// Loads the state from an AtomicUsize. + fn load(cell: &AtomicUsize, ordering: Ordering) -> SemState { + let value = cell.load(ordering); + SemState(value) + } -impl TryAcquireError { - /// Returns `true` if the error was caused by a closed semaphore. - pub(crate) fn is_closed(&self) -> bool { - match self { - TryAcquireError::Closed => true, - _ => false, - } + fn fetch_set_closed(cell: &AtomicUsize, ordering: Ordering) -> SemState { + let value = cell.fetch_or(CLOSED_FLAG, ordering); + SemState(value) } - /// Returns `true` if the error was caused by calling `try_acquire` on a - /// semaphore with no available permits. - pub(crate) fn is_no_permits(&self) -> bool { - match self { - TryAcquireError::NoPermits => true, - _ => false, - } + fn is_closed(self) -> bool { + self.0 & CLOSED_FLAG == CLOSED_FLAG + } + + /// Converts the state into a `usize` representation. + fn to_usize(self) -> usize { + self.0 } } -impl fmt::Display for TryAcquireError { +impl fmt::Debug for SemState { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - TryAcquireError::Closed => write!(fmt, "{}", "semaphore closed"), - TryAcquireError::NoPermits => write!(fmt, "{}", "no permits available"), + let mut fmt = fmt.debug_struct("SemState"); + + if self.is_waiter() { + fmt.field("state", &""); + } else { + fmt.field("permits", &self.available_permits()); } + + fmt.finish() } } -impl std::error::Error for TryAcquireError {} +// ===== impl WaiterState ===== -/// # Safety -/// -/// `Waiter` is forced to be !Unpin. -unsafe impl linked_list::Link for Waiter { - type Handle = *mut Waiter; - type Target = Waiter; +impl WaiterState { + fn permits_to_acquire(self) -> usize { + self.0 >> PERMIT_SHIFT + } + + fn set_permits_to_acquire(&mut self, val: usize) { + self.0 = (val << PERMIT_SHIFT) | (self.0 & !PERMIT_MASK) + } + + fn is_queued(self) -> bool { + self.0 & QUEUED == QUEUED + } + + fn set_queued(&mut self) { + self.0 |= QUEUED; + } + + fn is_closed(self) -> bool { + self.0 & CLOSED == CLOSED + } - fn to_raw(handle: *mut Waiter) -> NonNull { - debug_assert!(!handle.is_null()); - unsafe { NonNull::new_unchecked(handle) } + fn set_closed(&mut self) { + self.0 |= CLOSED; } - unsafe fn from_raw(ptr: NonNull) -> *mut Waiter { - ptr.as_ptr() + fn unset_queued(&mut self) { + assert!(self.is_queued()); + self.0 -= QUEUED; } - unsafe fn pointers(mut target: NonNull) -> NonNull> { - NonNull::from(&mut target.as_mut().pointers) + fn is_dropped(self) -> bool { + self.0 & DROPPED == DROPPED } } diff --git a/tokio/src/sync/semaphore_ll_old.rs b/tokio/src/sync/semaphore_ll_old.rs deleted file mode 100644 index 69fd4a6a5d3..00000000000 --- a/tokio/src/sync/semaphore_ll_old.rs +++ /dev/null @@ -1,1219 +0,0 @@ -#![cfg_attr(not(feature = "sync"), allow(dead_code, unreachable_pub))] - -//! Thread-safe, asynchronous counting semaphore. -//! -//! A `Semaphore` instance holds a set of permits. Permits are used to -//! synchronize access to a shared resource. -//! -//! Before accessing the shared resource, callers acquire a permit from the -//! semaphore. Once the permit is acquired, the caller then enters the critical -//! section. If no permits are available, then acquiring the semaphore returns -//! `Pending`. The task is woken once a permit becomes available. - -use crate::loom::cell::CausalCell; -use crate::loom::future::AtomicWaker; -use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize}; -use crate::loom::thread; - -use std::cmp; -use std::fmt; -use std::ptr::{self, NonNull}; -use std::sync::atomic::Ordering::{self, AcqRel, Acquire, Relaxed, Release}; -use std::task::Poll::{Pending, Ready}; -use std::task::{Context, Poll}; -use std::usize; - -/// Futures-aware semaphore. -pub(crate) struct Semaphore { - /// Tracks both the waiter queue tail pointer and the number of remaining - /// permits. - state: AtomicUsize, - - /// waiter queue head pointer. - head: CausalCell>, - - /// Coordinates access to the queue head. - rx_lock: AtomicUsize, - - /// Stub waiter node used as part of the MPSC channel algorithm. - stub: Box, -} - -/// A semaphore permit -/// -/// Tracks the lifecycle of a semaphore permit. -/// -/// An instance of `Permit` is intended to be used with a **single** instance of -/// `Semaphore`. Using a single instance of `Permit` with multiple semaphore -/// instances will result in unexpected behavior. -/// -/// `Permit` does **not** release the permit back to the semaphore on drop. It -/// is the user's responsibility to ensure that `Permit::release` is called -/// before dropping the permit. -#[derive(Debug)] -pub(crate) struct Permit { - waiter: Option>, - state: PermitState, -} - -/// Error returned by `Permit::poll_acquire`. -#[derive(Debug)] -pub(crate) struct AcquireError(()); - -/// Error returned by `Permit::try_acquire`. -#[derive(Debug)] -pub(crate) enum TryAcquireError { - Closed, - NoPermits, -} - -/// Node used to notify the semaphore waiter when permit is available. -#[derive(Debug)] -struct Waiter { - /// Stores waiter state. - /// - /// See `WaiterState` for more details. - state: AtomicUsize, - - /// Task to wake when a permit is made available. - waker: AtomicWaker, - - /// Next pointer in the queue of waiting senders. - next: AtomicPtr, -} - -/// Semaphore state -/// -/// The 2 low bits track the modes. -/// -/// - Closed -/// - Full -/// -/// When not full, the rest of the `usize` tracks the total number of messages -/// in the channel. When full, the rest of the `usize` is a pointer to the tail -/// of the "waiting senders" queue. -#[derive(Copy, Clone)] -struct SemState(usize); - -/// Permit state -#[derive(Debug, Copy, Clone)] -enum PermitState { - /// Currently waiting for permits to be made available and assigned to the - /// waiter. - Waiting(u16), - - /// The number of acquired permits - Acquired(u16), -} - -/// State for an individual waker node -#[derive(Debug, Copy, Clone)] -struct WaiterState(usize); - -/// Waiter node is in the semaphore queue -const QUEUED: usize = 0b001; - -/// Semaphore has been closed, no more permits will be issued. -const CLOSED: usize = 0b10; - -/// The permit that owns the `Waiter` dropped. -const DROPPED: usize = 0b100; - -/// Represents "one requested permit" in the waiter state -const PERMIT_ONE: usize = 0b1000; - -/// Masks the waiter state to only contain bits tracking number of requested -/// permits. -const PERMIT_MASK: usize = usize::MAX - (PERMIT_ONE - 1); - -/// How much to shift a permit count to pack it into the waker state -const PERMIT_SHIFT: u32 = PERMIT_ONE.trailing_zeros(); - -/// Flag differentiating between available permits and waiter pointers. -/// -/// If we assume pointers are properly aligned, then the least significant bit -/// will always be zero. So, we use that bit to track if the value represents a -/// number. -const NUM_FLAG: usize = 0b01; - -/// Signal the semaphore is closed -const CLOSED_FLAG: usize = 0b10; - -/// Maximum number of permits a semaphore can manage -const MAX_PERMITS: usize = usize::MAX >> NUM_SHIFT; - -/// When representing "numbers", the state has to be shifted this much (to get -/// rid of the flag bit). -const NUM_SHIFT: usize = 2; - -// ===== impl Semaphore ===== - -impl Semaphore { - /// Creates a new semaphore with the initial number of permits - /// - /// # Panics - /// - /// Panics if `permits` is zero. - pub(crate) fn new(permits: usize) -> Semaphore { - let stub = Box::new(Waiter::new()); - let ptr = NonNull::from(&*stub); - - // Allocations are aligned - debug_assert!(ptr.as_ptr() as usize & NUM_FLAG == 0); - - let state = SemState::new(permits, &stub); - - Semaphore { - state: AtomicUsize::new(state.to_usize()), - head: CausalCell::new(ptr), - rx_lock: AtomicUsize::new(0), - stub, - } - } - - /// Returns the current number of available permits - pub(crate) fn available_permits(&self) -> usize { - let curr = SemState(self.state.load(Acquire)); - curr.available_permits() - } - - /// Tries to acquire the requested number of permits, registering the waiter - /// if not enough permits are available. - fn poll_acquire( - &self, - cx: &mut Context<'_>, - num_permits: u16, - permit: &mut Permit, - ) -> Poll> { - self.poll_acquire2(num_permits, || { - let waiter = permit.waiter.get_or_insert_with(|| Box::new(Waiter::new())); - - waiter.waker.register_by_ref(cx.waker()); - - Some(NonNull::from(&**waiter)) - }) - } - - fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { - match self.poll_acquire2(num_permits, || None) { - Poll::Ready(res) => res.map_err(to_try_acquire), - Poll::Pending => Err(TryAcquireError::NoPermits), - } - } - - /// Polls for a permit - /// - /// Tries to acquire available permits first. If unable to acquire a - /// sufficient number of permits, the caller's waiter is pushed onto the - /// semaphore's wait queue. - fn poll_acquire2( - &self, - num_permits: u16, - mut get_waiter: F, - ) -> Poll> - where - F: FnMut() -> Option>, - { - let num_permits = num_permits as usize; - - // Load the current state - let mut curr = SemState(self.state.load(Acquire)); - - // Saves a ref to the waiter node - let mut maybe_waiter: Option> = None; - - /// Used in branches where we attempt to push the waiter into the wait - /// queue but fail due to permits becoming available or the wait queue - /// transitioning to "closed". In this case, the waiter must be - /// transitioned back to the "idle" state. - macro_rules! revert_to_idle { - () => { - if let Some(waiter) = maybe_waiter { - unsafe { waiter.as_ref() }.revert_to_idle(); - } - }; - } - - loop { - let mut next = curr; - - if curr.is_closed() { - revert_to_idle!(); - return Ready(Err(AcquireError::closed())); - } - - let acquired = next.acquire_permits(num_permits, &self.stub); - - if !acquired { - // There are not enough available permits to satisfy the - // request. The permit transitions to a waiting state. - debug_assert!(curr.waiter().is_some() || curr.available_permits() < num_permits); - - if let Some(waiter) = maybe_waiter.as_ref() { - // Safety: the caller owns the waiter. - let w = unsafe { waiter.as_ref() }; - w.set_permits_to_acquire(num_permits - curr.available_permits()); - } else { - // Get the waiter for the permit. - if let Some(waiter) = get_waiter() { - // Safety: the caller owns the waiter. - let w = unsafe { waiter.as_ref() }; - - // If there are any currently available permits, the - // waiter acquires those immediately and waits for the - // remaining permits to become available. - if !w.to_queued(num_permits - curr.available_permits()) { - // The node is alrady queued, there is no further work - // to do. - return Pending; - } - - maybe_waiter = Some(waiter); - } else { - // No waiter, this indicates the caller does not wish to - // "wait", so there is nothing left to do. - return Pending; - } - } - - next.set_waiter(maybe_waiter.unwrap()); - } - - debug_assert_ne!(curr.0, 0); - debug_assert_ne!(next.0, 0); - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => { - if acquired { - // Successfully acquire permits **without** queuing the - // waiter node. The waiter node is not currently in the - // queue. - revert_to_idle!(); - return Ready(Ok(())); - } else { - // The node is pushed into the queue, the final step is - // to set the node's "next" pointer to return the wait - // queue into a consistent state. - - let prev_waiter = - curr.waiter().unwrap_or_else(|| NonNull::from(&*self.stub)); - - let waiter = maybe_waiter.unwrap(); - - // Link the nodes. - // - // Safety: the mpsc algorithm guarantees the old tail of - // the queue is not removed from the queue during the - // push process. - unsafe { - prev_waiter.as_ref().store_next(waiter); - } - - return Pending; - } - } - Err(actual) => { - curr = SemState(actual); - } - } - } - } - - /// Closes the semaphore. This prevents the semaphore from issuing new - /// permits and notifies all pending waiters. - pub(crate) fn close(&self) { - // Acquire the `rx_lock`, setting the "closed" flag on the lock. - let prev = self.rx_lock.fetch_or(1, AcqRel); - - if prev != 0 { - // Another thread has the lock and will be responsible for notifying - // pending waiters. - return; - } - - self.add_permits_locked(0, true); - } - - /// Adds `n` new permits to the semaphore. - pub(crate) fn add_permits(&self, n: usize) { - if n == 0 { - return; - } - - // TODO: Handle overflow. A panic is not sufficient, the process must - // abort. - let prev = self.rx_lock.fetch_add(n << 1, AcqRel); - - if prev != 0 { - // Another thread has the lock and will be responsible for notifying - // pending waiters. - return; - } - - self.add_permits_locked(n, false); - } - - fn add_permits_locked(&self, mut rem: usize, mut closed: bool) { - while rem > 0 || closed { - if closed { - SemState::fetch_set_closed(&self.state, AcqRel); - } - - // Release the permits and notify - self.add_permits_locked2(rem, closed); - - let n = rem << 1; - - let actual = if closed { - let actual = self.rx_lock.fetch_sub(n | 1, AcqRel); - closed = false; - actual - } else { - let actual = self.rx_lock.fetch_sub(n, AcqRel); - closed = actual & 1 == 1; - actual - }; - - rem = (actual >> 1) - rem; - } - } - - /// Releases a specific amount of permits to the semaphore - /// - /// This function is called by `add_permits` after the add lock has been - /// acquired. - fn add_permits_locked2(&self, mut n: usize, closed: bool) { - // If closing the semaphore, we want to drain the entire queue. The - // number of permits being assigned doesn't matter. - if closed { - n = usize::MAX; - } - - 'outer: while n > 0 { - unsafe { - let mut head = self.head.with(|head| *head); - let mut next_ptr = head.as_ref().next.load(Acquire); - - let stub = self.stub(); - - if head == stub { - // The stub node indicates an empty queue. Any remaining - // permits get assigned back to the semaphore. - let next = match NonNull::new(next_ptr) { - Some(next) => next, - None => { - // This loop is not part of the standard intrusive mpsc - // channel algorithm. This is where we atomically pop - // the last task and add `n` to the remaining capacity. - // - // This modification to the pop algorithm works because, - // at this point, we have not done any work (only done - // reading). We have a *pretty* good idea that there is - // no concurrent pusher. - // - // The capacity is then atomically added by doing an - // AcqRel CAS on `state`. The `state` cell is the - // linchpin of the algorithm. - // - // By successfully CASing `head` w/ AcqRel, we ensure - // that, if any thread was racing and entered a push, we - // see that and abort pop, retrying as it is - // "inconsistent". - let mut curr = SemState::load(&self.state, Acquire); - - loop { - if curr.has_waiter(&self.stub) { - // A waiter is being added concurrently. - // This is the MPSC queue's "inconsistent" - // state and we must loop and try again. - thread::yield_now(); - continue 'outer; - } - - // If closing, nothing more to do. - if closed { - debug_assert!(curr.is_closed(), "state = {:?}", curr); - return; - } - - let mut next = curr; - next.release_permits(n, &self.stub); - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => return, - Err(actual) => { - curr = SemState(actual); - } - } - } - } - }; - - self.head.with_mut(|head| *head = next); - head = next; - next_ptr = next.as_ref().next.load(Acquire); - } - - // `head` points to a waiter assign permits to the waiter. If - // all requested permits are satisfied, then we can continue, - // otherwise the node stays in the wait queue. - if !head.as_ref().assign_permits(&mut n, closed) { - assert_eq!(n, 0); - return; - } - - if let Some(next) = NonNull::new(next_ptr) { - self.head.with_mut(|head| *head = next); - - self.remove_queued(head, closed); - continue 'outer; - } - - let state = SemState::load(&self.state, Acquire); - - // This must always be a pointer as the wait list is not empty. - let tail = state.waiter().unwrap(); - - if tail != head { - // Inconsistent - thread::yield_now(); - continue 'outer; - } - - self.push_stub(closed); - - next_ptr = head.as_ref().next.load(Acquire); - - if let Some(next) = NonNull::new(next_ptr) { - self.head.with_mut(|head| *head = next); - - self.remove_queued(head, closed); - continue 'outer; - } - - // Inconsistent state, loop - thread::yield_now(); - } - } - } - - /// The wait node has had all of its permits assigned and has been removed - /// from the wait queue. - /// - /// Attempt to remove the QUEUED bit from the node. If additional permits - /// are concurrently requested, the node must be pushed back into the wait - /// queued. - fn remove_queued(&self, waiter: NonNull, closed: bool) { - let mut curr = WaiterState(unsafe { waiter.as_ref() }.state.load(Acquire)); - - loop { - if curr.is_dropped() { - // The Permit dropped, it is on us to release the memory - let _ = unsafe { Box::from_raw(waiter.as_ptr()) }; - return; - } - - // The node is removed from the queue. We attempt to unset the - // queued bit, but concurrently the waiter has requested more - // permits. When the waiter requested more permits, it saw the - // queued bit set so took no further action. This requires us to - // push the node back into the queue. - if curr.permits_to_acquire() > 0 { - // More permits are requested. The waiter must be re-queued - unsafe { - self.push_waiter(waiter, closed); - } - return; - } - - let mut next = curr; - next.unset_queued(); - - let w = unsafe { waiter.as_ref() }; - - match w.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => return, - Err(actual) => { - curr = WaiterState(actual); - } - } - } - } - - unsafe fn push_stub(&self, closed: bool) { - self.push_waiter(self.stub(), closed); - } - - unsafe fn push_waiter(&self, waiter: NonNull, closed: bool) { - // Set the next pointer. This does not require an atomic operation as - // this node is not accessible. The write will be flushed with the next - // operation - waiter.as_ref().next.store(ptr::null_mut(), Relaxed); - - // Update the tail to point to the new node. We need to see the previous - // node in order to update the next pointer as well as release `task` - // to any other threads calling `push`. - let next = SemState::new_ptr(waiter, closed); - let prev = SemState(self.state.swap(next.0, AcqRel)); - - debug_assert_eq!(closed, prev.is_closed()); - - // This function is only called when there are pending tasks. Because of - // this, the state must *always* be in pointer mode. - let prev = prev.waiter().unwrap(); - - // No cycles plz - debug_assert_ne!(prev, waiter); - - // Release `task` to the consume end. - prev.as_ref().next.store(waiter.as_ptr(), Release); - } - - fn stub(&self) -> NonNull { - unsafe { NonNull::new_unchecked(&*self.stub as *const _ as *mut _) } - } -} - -impl Drop for Semaphore { - fn drop(&mut self) { - self.close(); - } -} - -impl fmt::Debug for Semaphore { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("Semaphore") - .field("state", &SemState::load(&self.state, Relaxed)) - .field("head", &self.head.with(|ptr| ptr)) - .field("rx_lock", &self.rx_lock.load(Relaxed)) - .field("stub", &self.stub) - .finish() - } -} - -unsafe impl Send for Semaphore {} -unsafe impl Sync for Semaphore {} - -// ===== impl Permit ===== - -impl Permit { - /// Creates a new `Permit`. - /// - /// The permit begins in the "unacquired" state. - pub(crate) fn new() -> Permit { - use PermitState::Acquired; - - Permit { - waiter: None, - state: Acquired(0), - } - } - - /// Returns `true` if the permit has been acquired - pub(crate) fn is_acquired(&self) -> bool { - match self.state { - PermitState::Acquired(num) if num > 0 => true, - _ => false, - } - } - - /// Tries to acquire the permit. If no permits are available, the current task - /// is notified once a new permit becomes available. - pub(crate) fn poll_acquire( - &mut self, - cx: &mut Context<'_>, - num_permits: u16, - semaphore: &Semaphore, - ) -> Poll> { - use std::cmp::Ordering::*; - use PermitState::*; - - match self.state { - Waiting(requested) => { - // There must be a waiter - let waiter = self.waiter.as_ref().unwrap(); - - match requested.cmp(&num_permits) { - Less => { - let delta = num_permits - requested; - - // Request additional permits. If the waiter has been - // dequeued, it must be re-queued. - if !waiter.try_inc_permits_to_acquire(delta as usize) { - let waiter = NonNull::from(&**waiter); - - // Ignore the result. The check for - // `permits_to_acquire()` will converge the state as - // needed - let _ = semaphore.poll_acquire2(delta, || Some(waiter))?; - } - - self.state = Waiting(num_permits); - } - Greater => { - let delta = requested - num_permits; - let to_release = waiter.try_dec_permits_to_acquire(delta as usize); - - semaphore.add_permits(to_release); - self.state = Waiting(num_permits); - } - Equal => {} - } - - if waiter.permits_to_acquire()? == 0 { - self.state = Acquired(requested); - return Ready(Ok(())); - } - - waiter.waker.register_by_ref(cx.waker()); - - if waiter.permits_to_acquire()? == 0 { - self.state = Acquired(requested); - return Ready(Ok(())); - } - - Pending - } - Acquired(acquired) => { - if acquired >= num_permits { - Ready(Ok(())) - } else { - match semaphore.poll_acquire(cx, num_permits - acquired, self)? { - Ready(()) => { - self.state = Acquired(num_permits); - Ready(Ok(())) - } - Pending => { - self.state = Waiting(num_permits); - Pending - } - } - } - } - } - } - - /// Tries to acquire the permit. - pub(crate) fn try_acquire( - &mut self, - num_permits: u16, - semaphore: &Semaphore, - ) -> Result<(), TryAcquireError> { - use PermitState::*; - - match self.state { - Waiting(requested) => { - // There must be a waiter - let waiter = self.waiter.as_ref().unwrap(); - - if requested > num_permits { - let delta = requested - num_permits; - let to_release = waiter.try_dec_permits_to_acquire(delta as usize); - - semaphore.add_permits(to_release); - self.state = Waiting(num_permits); - } - - let res = waiter.permits_to_acquire().map_err(to_try_acquire)?; - - if res == 0 { - if requested < num_permits { - // Try to acquire the additional permits - semaphore.try_acquire(num_permits - requested)?; - } - - self.state = Acquired(num_permits); - Ok(()) - } else { - Err(TryAcquireError::NoPermits) - } - } - Acquired(acquired) => { - if acquired < num_permits { - semaphore.try_acquire(num_permits - acquired)?; - self.state = Acquired(num_permits); - } - - Ok(()) - } - } - } - - /// Releases a permit back to the semaphore - pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { - let n = self.forget(n); - semaphore.add_permits(n as usize); - } - - /// Forgets the permit **without** releasing it back to the semaphore. - /// - /// After calling `forget`, `poll_acquire` is able to acquire new permit - /// from the sempahore. - /// - /// Repeatedly calling `forget` without associated calls to `add_permit` - /// will result in the semaphore losing all permits. - /// - /// Will forget **at most** the number of acquired permits. This number is - /// returned. - pub(crate) fn forget(&mut self, n: u16) -> u16 { - use PermitState::*; - - match self.state { - Waiting(requested) => { - let n = cmp::min(n, requested); - - // Decrement - let acquired = self - .waiter - .as_ref() - .unwrap() - .try_dec_permits_to_acquire(n as usize) as u16; - - if n == requested { - self.state = Acquired(0); - } else if acquired == requested - n { - self.state = Waiting(acquired); - } else { - self.state = Waiting(requested - n); - } - - acquired - } - Acquired(acquired) => { - let n = cmp::min(n, acquired); - self.state = Acquired(acquired - n); - n - } - } - } -} - -impl Default for Permit { - fn default() -> Self { - Self::new() - } -} - -impl Drop for Permit { - fn drop(&mut self) { - if let Some(waiter) = self.waiter.take() { - // Set the dropped flag - let state = WaiterState(waiter.state.fetch_or(DROPPED, AcqRel)); - - if state.is_queued() { - // The waiter is stored in the queue. The semaphore will drop it - std::mem::forget(waiter); - } - } - } -} - -// ===== impl AcquireError ==== - -impl AcquireError { - fn closed() -> AcquireError { - AcquireError(()) - } -} - -fn to_try_acquire(_: AcquireError) -> TryAcquireError { - TryAcquireError::Closed -} - -impl fmt::Display for AcquireError { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(fmt, "semaphore closed") - } -} - -impl std::error::Error for AcquireError {} - -// ===== impl TryAcquireError ===== - -impl TryAcquireError { - /// Returns `true` if the error was caused by a closed semaphore. - pub(crate) fn is_closed(&self) -> bool { - match self { - TryAcquireError::Closed => true, - _ => false, - } - } - - /// Returns `true` if the error was caused by calling `try_acquire` on a - /// semaphore with no available permits. - pub(crate) fn is_no_permits(&self) -> bool { - match self { - TryAcquireError::NoPermits => true, - _ => false, - } - } -} - -impl fmt::Display for TryAcquireError { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - TryAcquireError::Closed => write!(fmt, "{}", "semaphore closed"), - TryAcquireError::NoPermits => write!(fmt, "{}", "no permits available"), - } - } -} - -impl std::error::Error for TryAcquireError {} - -// ===== impl Waiter ===== - -impl Waiter { - fn new() -> Waiter { - Waiter { - state: AtomicUsize::new(0), - waker: AtomicWaker::new(), - next: AtomicPtr::new(ptr::null_mut()), - } - } - - fn permits_to_acquire(&self) -> Result { - let state = WaiterState(self.state.load(Acquire)); - - if state.is_closed() { - Err(AcquireError(())) - } else { - Ok(state.permits_to_acquire()) - } - } - - /// Only increments the number of permits *if* the waiter is currently - /// queued. - /// - /// # Returns - /// - /// `true` if the number of permits to acquire has been incremented. `false` - /// otherwise. On `false`, the caller should use `Semaphore::poll_acquire`. - fn try_inc_permits_to_acquire(&self, n: usize) -> bool { - let mut curr = WaiterState(self.state.load(Acquire)); - - loop { - if !curr.is_queued() { - assert_eq!(0, curr.permits_to_acquire()); - return false; - } - - let mut next = curr; - next.set_permits_to_acquire(n + curr.permits_to_acquire()); - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => return true, - Err(actual) => curr = WaiterState(actual), - } - } - } - - /// Try to decrement the number of permits to acquire. This returns the - /// actual number of permits that were decremented. The delta betweeen `n` - /// and the return has been assigned to the permit and the caller must - /// assign these back to the semaphore. - fn try_dec_permits_to_acquire(&self, n: usize) -> usize { - let mut curr = WaiterState(self.state.load(Acquire)); - - loop { - if !curr.is_queued() { - assert_eq!(0, curr.permits_to_acquire()); - } - - let delta = cmp::min(n, curr.permits_to_acquire()); - let rem = curr.permits_to_acquire() - delta; - - let mut next = curr; - next.set_permits_to_acquire(rem); - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => return n - delta, - Err(actual) => curr = WaiterState(actual), - } - } - } - - /// Store the number of remaining permits needed to satisfy the waiter and - /// transition to the "QUEUED" state. - /// - /// # Returns - /// - /// `true` if the `QUEUED` bit was set as part of the transition. - fn to_queued(&self, num_permits: usize) -> bool { - let mut curr = WaiterState(self.state.load(Acquire)); - - // The waiter should **not** be waiting for any permits. - debug_assert_eq!(curr.permits_to_acquire(), 0); - - loop { - let mut next = curr; - next.set_permits_to_acquire(num_permits); - next.set_queued(); - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => { - if curr.is_queued() { - return false; - } else { - // Make sure the next pointer is null - self.next.store(ptr::null_mut(), Relaxed); - return true; - } - } - Err(actual) => curr = WaiterState(actual), - } - } - } - - /// Set the number of permits to acquire. - /// - /// This function is only called when the waiter is being inserted into the - /// wait queue. Because of this, there are no concurrent threads that can - /// modify the state and using `store` is safe. - fn set_permits_to_acquire(&self, num_permits: usize) { - debug_assert!(WaiterState(self.state.load(Acquire)).is_queued()); - - let mut state = WaiterState(QUEUED); - state.set_permits_to_acquire(num_permits); - - self.state.store(state.0, Release); - } - - /// Assign permits to the waiter. - /// - /// Returns `true` if the waiter should be removed from the queue - fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { - let mut curr = WaiterState(self.state.load(Acquire)); - - loop { - let mut next = curr; - - // Number of permits to assign to this waiter - let assign = cmp::min(curr.permits_to_acquire(), *n); - - // Assign the permits - next.set_permits_to_acquire(curr.permits_to_acquire() - assign); - - if closed { - next.set_closed(); - } - - match self.state.compare_exchange(curr.0, next.0, AcqRel, Acquire) { - Ok(_) => { - // Update `n` - *n -= assign; - - if next.permits_to_acquire() == 0 { - if curr.permits_to_acquire() > 0 { - self.waker.wake(); - } - - return true; - } else { - return false; - } - } - Err(actual) => curr = WaiterState(actual), - } - } - } - - fn revert_to_idle(&self) { - // An idle node is not waiting on any permits - self.state.store(0, Relaxed); - } - - fn store_next(&self, next: NonNull) { - self.next.store(next.as_ptr(), Release); - } -} - -// ===== impl SemState ===== - -impl SemState { - /// Returns a new default `State` value. - fn new(permits: usize, stub: &Waiter) -> SemState { - assert!(permits <= MAX_PERMITS); - - if permits > 0 { - SemState((permits << NUM_SHIFT) | NUM_FLAG) - } else { - SemState(stub as *const _ as usize) - } - } - - /// Returns a `State` tracking `ptr` as the tail of the queue. - fn new_ptr(tail: NonNull, closed: bool) -> SemState { - let mut val = tail.as_ptr() as usize; - - if closed { - val |= CLOSED_FLAG; - } - - SemState(val) - } - - /// Returns the amount of remaining capacity - fn available_permits(self) -> usize { - if !self.has_available_permits() { - return 0; - } - - self.0 >> NUM_SHIFT - } - - /// Returns `true` if the state has permits that can be claimed by a waiter. - fn has_available_permits(self) -> bool { - self.0 & NUM_FLAG == NUM_FLAG - } - - fn has_waiter(self, stub: &Waiter) -> bool { - !self.has_available_permits() && !self.is_stub(stub) - } - - /// Tries to atomically acquire specified number of permits. - /// - /// # Return - /// - /// Returns `true` if the specified number of permits were acquired, `false` - /// otherwise. Returning false does not mean that there are no more - /// available permits. - fn acquire_permits(&mut self, num: usize, stub: &Waiter) -> bool { - debug_assert!(num > 0); - - if self.available_permits() < num { - return false; - } - - debug_assert!(self.waiter().is_none()); - - self.0 -= num << NUM_SHIFT; - - if self.0 == NUM_FLAG { - // Set the state to the stub pointer. - self.0 = stub as *const _ as usize; - } - - true - } - - /// Releases permits - /// - /// Returns `true` if the permits were accepted. - fn release_permits(&mut self, permits: usize, stub: &Waiter) { - debug_assert!(permits > 0); - - if self.is_stub(stub) { - self.0 = (permits << NUM_SHIFT) | NUM_FLAG | (self.0 & CLOSED_FLAG); - return; - } - - debug_assert!(self.has_available_permits()); - - self.0 += permits << NUM_SHIFT; - } - - fn is_waiter(self) -> bool { - self.0 & NUM_FLAG == 0 - } - - /// Returns the waiter, if one is set. - fn waiter(self) -> Option> { - if self.is_waiter() { - let waiter = NonNull::new(self.as_ptr()).expect("null pointer stored"); - - Some(waiter) - } else { - None - } - } - - /// Assumes `self` represents a pointer - fn as_ptr(self) -> *mut Waiter { - (self.0 & !CLOSED_FLAG) as *mut Waiter - } - - /// Sets to a pointer to a waiter. - /// - /// This can only be done from the full state. - fn set_waiter(&mut self, waiter: NonNull) { - let waiter = waiter.as_ptr() as usize; - debug_assert!(!self.is_closed()); - - self.0 = waiter; - } - - fn is_stub(self, stub: &Waiter) -> bool { - self.as_ptr() as usize == stub as *const _ as usize - } - - /// Loads the state from an AtomicUsize. - fn load(cell: &AtomicUsize, ordering: Ordering) -> SemState { - let value = cell.load(ordering); - SemState(value) - } - - fn fetch_set_closed(cell: &AtomicUsize, ordering: Ordering) -> SemState { - let value = cell.fetch_or(CLOSED_FLAG, ordering); - SemState(value) - } - - fn is_closed(self) -> bool { - self.0 & CLOSED_FLAG == CLOSED_FLAG - } - - /// Converts the state into a `usize` representation. - fn to_usize(self) -> usize { - self.0 - } -} - -impl fmt::Debug for SemState { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut fmt = fmt.debug_struct("SemState"); - - if self.is_waiter() { - fmt.field("state", &""); - } else { - fmt.field("permits", &self.available_permits()); - } - - fmt.finish() - } -} - -// ===== impl WaiterState ===== - -impl WaiterState { - fn permits_to_acquire(self) -> usize { - self.0 >> PERMIT_SHIFT - } - - fn set_permits_to_acquire(&mut self, val: usize) { - self.0 = (val << PERMIT_SHIFT) | (self.0 & !PERMIT_MASK) - } - - fn is_queued(self) -> bool { - self.0 & QUEUED == QUEUED - } - - fn set_queued(&mut self) { - self.0 |= QUEUED; - } - - fn is_closed(self) -> bool { - self.0 & CLOSED == CLOSED - } - - fn set_closed(&mut self) { - self.0 |= CLOSED; - } - - fn unset_queued(&mut self) { - assert!(self.is_queued()); - self.0 -= QUEUED; - } - - fn is_dropped(self) -> bool { - self.0 & DROPPED == DROPPED - } -} diff --git a/tokio/src/sync/tests/mod.rs b/tokio/src/sync/tests/mod.rs index 7225ce9c58c..32926e49375 100644 --- a/tokio/src/sync/tests/mod.rs +++ b/tokio/src/sync/tests/mod.rs @@ -1,6 +1,7 @@ cfg_not_loom! { mod atomic_waker; mod semaphore_ll; + mod semaphore_batch; } cfg_loom! { diff --git a/tokio/src/sync/tests/semaphore_batch.rs b/tokio/src/sync/tests/semaphore_batch.rs new file mode 100644 index 00000000000..776cf1aa4d3 --- /dev/null +++ b/tokio/src/sync/tests/semaphore_batch.rs @@ -0,0 +1,480 @@ +use crate::sync::batch_semaphore::{Permit, Semaphore}; +use tokio_test::*; + +#[test] +fn poll_acquire_one_available() { + let s = Semaphore::new(100); + assert_eq!(s.available_permits(), 100); + + // Polling for a permit succeeds immediately + let mut permit = Permit::new(); + assert!(!permit.is_acquired()); + + assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + assert_eq!(s.available_permits(), 99); + assert!(permit.is_acquired()); + + // Polling again on the same waiter does not claim a new permit + assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + assert_eq!(s.available_permits(), 99); + assert!(permit.is_acquired()); +} + +#[test] +fn poll_acquire_many_available() { + let s = Semaphore::new(100); + assert_eq!(s.available_permits(), 100); + + // Polling for a permit succeeds immediately + let mut permit = Permit::new(); + assert!(!permit.is_acquired()); + + assert_ready_ok!(task::spawn(permit.acquire(5, &s)).poll()); + assert_eq!(s.available_permits(), 95); + assert!(permit.is_acquired()); + + // Polling again on the same waiter does not claim a new permit + assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + assert_eq!(s.available_permits(), 95); + assert!(permit.is_acquired()); + + assert_ready_ok!(task::spawn(permit.acquire(5, &s)).poll()); + assert_eq!(s.available_permits(), 95); + assert!(permit.is_acquired()); + + // Polling for a larger number of permits acquires more + assert_ready_ok!(task::spawn(permit.acquire(8, &s)).poll()); + assert_eq!(s.available_permits(), 92); + assert!(permit.is_acquired()); +} + +#[test] +fn try_acquire_one_available() { + let s = Semaphore::new(100); + assert_eq!(s.available_permits(), 100); + + // Polling for a permit succeeds immediately + let mut permit = Permit::new(); + assert!(!permit.is_acquired()); + + assert_ok!(permit.try_acquire(1, &s)); + assert_eq!(s.available_permits(), 99); + assert!(permit.is_acquired()); + + // Polling again on the same waiter does not claim a new permit + assert_ok!(permit.try_acquire(1, &s)); + assert_eq!(s.available_permits(), 99); + assert!(permit.is_acquired()); +} + +#[test] +fn try_acquire_many_available() { + let s = Semaphore::new(100); + assert_eq!(s.available_permits(), 100); + + // Polling for a permit succeeds immediately + let mut permit = Permit::new(); + assert!(!permit.is_acquired()); + + assert_ok!(permit.try_acquire(5, &s)); + assert_eq!(s.available_permits(), 95); + assert!(permit.is_acquired()); + + // Polling again on the same waiter does not claim a new permit + assert_ok!(permit.try_acquire(5, &s)); + assert_eq!(s.available_permits(), 95); + assert!(permit.is_acquired()); +} + +#[test] + +fn poll_acquire_one_unavailable() { + let s = Semaphore::new(1); + + let mut permit_1 = Permit::new(); + let mut permit_2 = Permit::new(); + + { + let mut acquire_1 = task::spawn(permit_1.acquire(1, &s)); + // Acquire the first permit + assert_ready_ok!(acquire_1.poll()); + assert_eq!(s.available_permits(), 0); + } + + { + let mut acquire_2 = task::spawn(permit_2.acquire(1, &s)); + // Try to acquire the second permit + assert_pending!(acquire_2.poll()); + assert_eq!(s.available_permits(), 0); + + permit_1.release(1, &s); + + assert_eq!(s.available_permits(), 0); + assert!(acquire_2.is_woken()); + assert_ready_ok!(acquire_2.poll()); + assert_eq!(s.available_permits(), 0); + } + + assert_ready_ok!(task::spawn(permit_2.acquire(1, &s)).poll()); + + permit_2.release(1, &s); + assert_eq!(s.available_permits(), 1); +} + +#[test] +fn forget_acquired() { + let s = Semaphore::new(1); + + // Polling for a permit succeeds immediately + let mut permit = Permit::new(); + + assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + + assert_eq!(s.available_permits(), 0); + + permit.forget(1, &s); + assert_eq!(s.available_permits(), 0); +} + +// #[test] +// fn forget_waiting() { +// let s = Semaphore::new(0); + +// // Polling for a permit succeeds immediately +// let mut permit = task::spawn(Box::new(Permit::new())); + +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + +// assert_eq!(s.available_permits(), 0); + +// permit.forget(1); + +// s.add_permits(1); + +// assert!(!permit.is_woken()); +// assert_eq!(s.available_permits(), 1); +// } + +#[test] +fn poll_acquire_many_unavailable() { + let s = Semaphore::new(5); + + let mut permit_1 = Permit::new(); + let mut permit_2 = Permit::new(); + let mut permit_3 = Permit::new(); + + // Acquire the first permit + assert_ready_ok!(task::spawn(permit_1.acquire(1, &s)).poll()); + assert_eq!(s.available_permits(), 4); + + // Try to acquire the second permit + let mut acquire_2 = task::spawn(permit_2.acquire(5, &s)); + assert_pending!(acquire_2.poll()); + assert_eq!(s.available_permits(), 0); + + // Try to acquire the third permit + let mut acquire_3 = task::spawn(permit_3.acquire(3, &s)); + assert_pending!(acquire_3.poll()); + assert_eq!(s.available_permits(), 0); + + permit_1.release(1, &s); + + assert_eq!(s.available_permits(), 0); + assert!(acquire_2.is_woken()); + assert_ready_ok!(acquire_2.poll()); + + assert!(!acquire_3.is_woken()); + assert_eq!(s.available_permits(), 0); + + drop(acquire_2); // drop the acquire future so we can now + permit_2.release(1, &s); + assert!(!acquire_3.is_woken()); + assert_eq!(s.available_permits(), 0); + + permit_2.release(2, &s); + assert!(acquire_3.is_woken()); + + assert_ready_ok!(acquire_3.poll()); +} + +// #[test] +// fn try_acquire_one_unavailable() { +// let s = Semaphore::new(1); + +// let mut permit_1 = Permit::new(); +// let mut permit_2 = Permit::new(); + +// // Acquire the first permit +// assert_ok!(permit_1.try_acquire(1, &s)); +// assert_eq!(s.available_permits(), 0); + +// assert_err!(permit_2.try_acquire(1, &s)); + +// permit_1.release(1, &s); + +// assert_eq!(s.available_permits(), 1); +// assert_ok!(permit_2.try_acquire(1, &s)); + +// permit_2.release(1, &s); +// assert_eq!(s.available_permits(), 1); +// } + +// #[test] +// fn try_acquire_many_unavailable() { +// let s = Semaphore::new(5); + +// let mut permit_1 = Permit::new(); +// let mut permit_2 = Permit::new(); + +// // Acquire the first permit +// assert_ok!(permit_1.try_acquire(1, &s)); +// assert_eq!(s.available_permits(), 4); + +// assert_err!(permit_2.try_acquire(5, &s)); + +// permit_1.release(1, &s); +// assert_eq!(s.available_permits(), 5); + +// assert_ok!(permit_2.try_acquire(5, &s)); + +// permit_2.release(1, &s); +// assert_eq!(s.available_permits(), 1); + +// permit_2.release(1, &s); +// assert_eq!(s.available_permits(), 2); +// } + +#[test] +fn poll_acquire_one_zero_permits() { + let s = Semaphore::new(0); + assert_eq!(s.available_permits(), 0); + + let mut permit = Permit::new(); + + // Try to acquire the permit + let mut acquire = task::spawn(permit.acquire(1, &s)); + assert_pending!(acquire.poll()); + + s.add_permits(1); + + assert!(acquire.is_woken()); + assert_ready_ok!(acquire.poll()); +} + +#[test] +#[should_panic] +fn validates_max_permits() { + use std::usize; + Semaphore::new((usize::MAX >> 2) + 1); +} + +#[test] +fn close_semaphore_prevents_acquire() { + let s = Semaphore::new(5); + s.close(); + + assert_eq!(5, s.available_permits()); + + let mut permit_1 = Permit::new(); + let mut permit_2 = Permit::new(); + + assert_ready_err!(task::spawn(permit_1.acquire(1, &s)).poll()); + assert_eq!(5, s.available_permits()); + + assert_ready_err!(task::spawn(permit_2.acquire(1, &s)).poll()); + assert_eq!(5, s.available_permits()); +} + +#[test] +fn close_semaphore_notifies_permit1() { + let s = Semaphore::new(0); + let mut permit = Permit::new(); + let mut acquire = task::spawn(permit.acquire(1, &s)); + + assert_pending!(acquire.poll()); + + s.close(); + + assert!(acquire.is_woken()); + assert_ready_err!(acquire.poll()); +} + +#[test] +fn close_semaphore_notifies_permit2() { + let s = Semaphore::new(2); + + let mut permit1 = Permit::new(); + let mut permit2 = Permit::new(); + let mut permit3 = Permit::new(); + let mut permit4 = Permit::new(); + + // Acquire a couple of permits + assert_ready_ok!(task::spawn(permit1.acquire(1, &s)).poll()); + assert_ready_ok!(task::spawn(permit2.acquire(1, &s)).poll()); + + let mut acquire3 = task::spawn(permit3.acquire(1, &s)); + let mut acquire4 = task::spawn(permit4.acquire(1, &s)); + assert_pending!(acquire3.poll()); + assert_pending!(acquire4.poll()); + + s.close(); + + assert!(acquire3.is_woken()); + assert!(acquire4.is_woken()); + + assert_ready_err!(acquire3.poll()); + assert_ready_err!(acquire4.poll()); + + assert_eq!(0, s.available_permits()); + + permit1.release(1, &s); + + assert_eq!(1, s.available_permits()); + + assert_ready_err!(task::spawn(permit1.acquire(1, &s)).poll()); + + permit2.release(1, &s); + + assert_eq!(2, s.available_permits()); +} + +// #[test] +// fn poll_acquire_additional_permits_while_waiting_before_assigned() { +// let s = Semaphore::new(1); + +// let mut permit = task::spawn(Box::new(Permit::new())); + +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); + +// s.add_permits(1); +// assert!(!permit.is_woken()); + +// s.add_permits(1); +// assert!(permit.is_woken()); + +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); +// } + +// #[test] +// fn try_acquire_additional_permits_while_waiting_before_assigned() { +// let s = Semaphore::new(1); + +// let mut permit = task::spawn(Box::new(Permit::new())); + +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); + +// assert_err!(permit.enter(|_, mut p| p.try_acquire(3, &s))); + +// s.add_permits(1); +// assert!(permit.is_woken()); + +// assert_ok!(permit.enter(|_, mut p| p.try_acquire(2, &s))); +// } + +// #[test] +// fn poll_acquire_additional_permits_while_waiting_after_assigned_success() { +// let s = Semaphore::new(1); + +// let mut permit = task::spawn(Box::new(Permit::new())); + +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); + +// s.add_permits(2); + +// assert!(permit.is_woken()); +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); +// } + +// #[test] +// fn poll_acquire_additional_permits_while_waiting_after_assigned_requeue() { +// let s = Semaphore::new(1); + +// let mut permit = task::spawn(Box::new(Permit::new())); + +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); + +// s.add_permits(2); + +// assert!(permit.is_woken()); +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); + +// s.add_permits(1); + +// assert!(permit.is_woken()); +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); +// } + +// #[test] +// fn poll_acquire_fewer_permits_while_waiting() { +// let s = Semaphore::new(1); + +// let mut permit = task::spawn(Box::new(Permit::new())); + +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// assert_eq!(s.available_permits(), 0); + +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// assert_eq!(s.available_permits(), 0); +// } + +// #[test] +// fn poll_acquire_fewer_permits_after_assigned() { +// let s = Semaphore::new(1); + +// let mut permit1 = task::spawn(Box::new(Permit::new())); +// let mut permit2 = task::spawn(Box::new(Permit::new())); + +// assert_pending!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); +// assert_eq!(s.available_permits(), 0); + +// assert_pending!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + +// s.add_permits(4); +// assert!(permit1.is_woken()); +// assert!(!permit2.is_woken()); + +// assert_ready_ok!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); + +// assert!(permit2.is_woken()); +// assert_eq!(s.available_permits(), 1); + +// assert_ready_ok!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); +// } + +// #[test] +// fn forget_partial_1() { +// let s = Semaphore::new(0); + +// let mut permit = task::spawn(Box::new(Permit::new())); + +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// s.add_permits(1); + +// assert_eq!(0, s.available_permits()); + +// permit.release(1, &s); + +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + +// assert_eq!(s.available_permits(), 0); +// } + +// #[test] +// fn forget_partial_2() { +// let s = Semaphore::new(0); + +// let mut permit = task::spawn(Box::new(Permit::new())); + +// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// s.add_permits(1); + +// assert_eq!(0, s.available_permits()); + +// permit.release(1, &s); + +// s.add_permits(1); + +// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); +// assert_eq!(s.available_permits(), 0); +// } diff --git a/tokio/src/sync/tests/semaphore_ll.rs b/tokio/src/sync/tests/semaphore_ll.rs index d7b2be161d8..bfb075780bb 100644 --- a/tokio/src/sync/tests/semaphore_ll.rs +++ b/tokio/src/sync/tests/semaphore_ll.rs @@ -7,15 +7,15 @@ fn poll_acquire_one_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = Permit::new(); + let mut permit = task::spawn(Permit::new()); assert!(!permit.is_acquired()); - assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 99); assert!(permit.is_acquired()); // Polling again on the same waiter does not claim a new permit - assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 99); assert!(permit.is_acquired()); } @@ -26,24 +26,24 @@ fn poll_acquire_many_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = Permit::new(); + let mut permit = task::spawn(Permit::new()); assert!(!permit.is_acquired()); - assert_ready_ok!(task::spawn(permit.acquire(5, &s)).poll()); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 5, &s))); assert_eq!(s.available_permits(), 95); assert!(permit.is_acquired()); // Polling again on the same waiter does not claim a new permit - assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 95); assert!(permit.is_acquired()); - assert_ready_ok!(task::spawn(permit.acquire(5, &s)).poll()); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 5, &s))); assert_eq!(s.available_permits(), 95); assert!(permit.is_acquired()); // Polling for a larger number of permits acquires more - assert_ready_ok!(task::spawn(permit.acquire(8, &s)).poll()); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 8, &s))); assert_eq!(s.available_permits(), 92); assert!(permit.is_acquired()); } @@ -87,35 +87,26 @@ fn try_acquire_many_available() { } #[test] - fn poll_acquire_one_unavailable() { let s = Semaphore::new(1); - let mut permit_1 = Permit::new(); - let mut permit_2 = Permit::new(); + let mut permit_1 = task::spawn(Permit::new()); + let mut permit_2 = task::spawn(Permit::new()); - { - let mut acquire_1 = task::spawn(permit_1.acquire(1, &s)); - // Acquire the first permit - assert_ready_ok!(acquire_1.poll()); - assert_eq!(s.available_permits(), 0); - } + // Acquire the first permit + assert_ready_ok!(permit_1.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_eq!(s.available_permits(), 0); - { - let mut acquire_2 = task::spawn(permit_2.acquire(1, &s)); + permit_2.enter(|cx, mut p| { // Try to acquire the second permit - assert_pending!(acquire_2.poll()); - assert_eq!(s.available_permits(), 0); - - permit_1.release(1, &s); + assert_pending!(p.poll_acquire(cx, 1, &s)); + }); - assert_eq!(s.available_permits(), 0); - assert!(acquire_2.is_woken()); - assert_ready_ok!(acquire_2.poll()); - assert_eq!(s.available_permits(), 0); - } + permit_1.release(1, &s); - assert_ready_ok!(task::spawn(permit_2.acquire(1, &s)).poll()); + assert_eq!(s.available_permits(), 0); + assert!(permit_2.is_woken()); + assert_ready_ok!(permit_2.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); permit_2.release(1, &s); assert_eq!(s.available_permits(), 1); @@ -126,139 +117,141 @@ fn forget_acquired() { let s = Semaphore::new(1); // Polling for a permit succeeds immediately - let mut permit = Permit::new(); + let mut permit = task::spawn(Permit::new()); - assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 0); - permit.forget(1, &s); + permit.forget(1); assert_eq!(s.available_permits(), 0); } -// #[test] -// fn forget_waiting() { -// let s = Semaphore::new(0); +#[test] +fn forget_waiting() { + let s = Semaphore::new(0); -// // Polling for a permit succeeds immediately -// let mut permit = task::spawn(Box::new(Permit::new())); + // Polling for a permit succeeds immediately + let mut permit = task::spawn(Permit::new()); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); -// assert_eq!(s.available_permits(), 0); + assert_eq!(s.available_permits(), 0); -// permit.forget(1); + permit.forget(1); -// s.add_permits(1); + s.add_permits(1); -// assert!(!permit.is_woken()); -// assert_eq!(s.available_permits(), 1); -// } + assert!(!permit.is_woken()); + assert_eq!(s.available_permits(), 1); +} #[test] fn poll_acquire_many_unavailable() { let s = Semaphore::new(5); - let mut permit_1 = Permit::new(); - let mut permit_2 = Permit::new(); - let mut permit_3 = Permit::new(); + let mut permit_1 = task::spawn(Permit::new()); + let mut permit_2 = task::spawn(Permit::new()); + let mut permit_3 = task::spawn(Permit::new()); // Acquire the first permit - assert_ready_ok!(task::spawn(permit_1.acquire(1, &s)).poll()); + assert_ready_ok!(permit_1.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); assert_eq!(s.available_permits(), 4); - // Try to acquire the second permit - let mut acquire_2 = task::spawn(permit_2.acquire(5, &s)); - assert_pending!(acquire_2.poll()); - assert_eq!(s.available_permits(), 0); + permit_2.enter(|cx, mut p| { + // Try to acquire the second permit + assert_pending!(p.poll_acquire(cx, 5, &s)); + }); - // Try to acquire the third permit - let mut acquire_3 = task::spawn(permit_3.acquire(3, &s)); - assert_pending!(acquire_3.poll()); assert_eq!(s.available_permits(), 0); + permit_3.enter(|cx, mut p| { + // Try to acquire the third permit + assert_pending!(p.poll_acquire(cx, 3, &s)); + }); + permit_1.release(1, &s); assert_eq!(s.available_permits(), 0); - assert!(acquire_2.is_woken()); - assert_ready_ok!(acquire_2.poll()); + assert!(permit_2.is_woken()); + assert_ready_ok!(permit_2.enter(|cx, mut p| p.poll_acquire(cx, 5, &s))); - assert!(!acquire_3.is_woken()); + assert!(!permit_3.is_woken()); assert_eq!(s.available_permits(), 0); - drop(acquire_2); // drop the acquire future so we can now permit_2.release(1, &s); - assert!(!acquire_3.is_woken()); + assert!(!permit_3.is_woken()); assert_eq!(s.available_permits(), 0); permit_2.release(2, &s); - assert!(acquire_3.is_woken()); + assert!(permit_3.is_woken()); - assert_ready_ok!(acquire_3.poll()); + assert_ready_ok!(permit_3.enter(|cx, mut p| p.poll_acquire(cx, 3, &s))); } -// #[test] -// fn try_acquire_one_unavailable() { -// let s = Semaphore::new(1); +#[test] +fn try_acquire_one_unavailable() { + let s = Semaphore::new(1); -// let mut permit_1 = Permit::new(); -// let mut permit_2 = Permit::new(); + let mut permit_1 = Permit::new(); + let mut permit_2 = Permit::new(); -// // Acquire the first permit -// assert_ok!(permit_1.try_acquire(1, &s)); -// assert_eq!(s.available_permits(), 0); + // Acquire the first permit + assert_ok!(permit_1.try_acquire(1, &s)); + assert_eq!(s.available_permits(), 0); -// assert_err!(permit_2.try_acquire(1, &s)); + assert_err!(permit_2.try_acquire(1, &s)); -// permit_1.release(1, &s); + permit_1.release(1, &s); -// assert_eq!(s.available_permits(), 1); -// assert_ok!(permit_2.try_acquire(1, &s)); + assert_eq!(s.available_permits(), 1); + assert_ok!(permit_2.try_acquire(1, &s)); -// permit_2.release(1, &s); -// assert_eq!(s.available_permits(), 1); -// } + permit_2.release(1, &s); + assert_eq!(s.available_permits(), 1); +} -// #[test] -// fn try_acquire_many_unavailable() { -// let s = Semaphore::new(5); +#[test] +fn try_acquire_many_unavailable() { + let s = Semaphore::new(5); -// let mut permit_1 = Permit::new(); -// let mut permit_2 = Permit::new(); + let mut permit_1 = Permit::new(); + let mut permit_2 = Permit::new(); -// // Acquire the first permit -// assert_ok!(permit_1.try_acquire(1, &s)); -// assert_eq!(s.available_permits(), 4); + // Acquire the first permit + assert_ok!(permit_1.try_acquire(1, &s)); + assert_eq!(s.available_permits(), 4); -// assert_err!(permit_2.try_acquire(5, &s)); + assert_err!(permit_2.try_acquire(5, &s)); -// permit_1.release(1, &s); -// assert_eq!(s.available_permits(), 5); + permit_1.release(1, &s); + assert_eq!(s.available_permits(), 5); -// assert_ok!(permit_2.try_acquire(5, &s)); + assert_ok!(permit_2.try_acquire(5, &s)); -// permit_2.release(1, &s); -// assert_eq!(s.available_permits(), 1); + permit_2.release(1, &s); + assert_eq!(s.available_permits(), 1); -// permit_2.release(1, &s); -// assert_eq!(s.available_permits(), 2); -// } + permit_2.release(1, &s); + assert_eq!(s.available_permits(), 2); +} #[test] fn poll_acquire_one_zero_permits() { let s = Semaphore::new(0); assert_eq!(s.available_permits(), 0); - let mut permit = Permit::new(); + let mut permit = task::spawn(Permit::new()); // Try to acquire the permit - let mut acquire = task::spawn(permit.acquire(1, &s)); - assert_pending!(acquire.poll()); + permit.enter(|cx, mut p| { + assert_pending!(p.poll_acquire(cx, 1, &s)); + }); s.add_permits(1); - assert!(acquire.is_woken()); - assert_ready_ok!(acquire.poll()); + assert!(permit.is_woken()); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); } #[test] @@ -275,55 +268,52 @@ fn close_semaphore_prevents_acquire() { assert_eq!(5, s.available_permits()); - let mut permit_1 = Permit::new(); - let mut permit_2 = Permit::new(); + let mut permit_1 = task::spawn(Permit::new()); + let mut permit_2 = task::spawn(Permit::new()); - assert_ready_err!(task::spawn(permit_1.acquire(1, &s)).poll()); + assert_ready_err!(permit_1.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); assert_eq!(5, s.available_permits()); - assert_ready_err!(task::spawn(permit_2.acquire(1, &s)).poll()); + assert_ready_err!(permit_2.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); assert_eq!(5, s.available_permits()); } #[test] fn close_semaphore_notifies_permit1() { let s = Semaphore::new(0); - let mut permit = Permit::new(); - let mut acquire = task::spawn(permit.acquire(1, &s)); + let mut permit = task::spawn(Permit::new()); - assert_pending!(acquire.poll()); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); s.close(); - assert!(acquire.is_woken()); - assert_ready_err!(acquire.poll()); + assert!(permit.is_woken()); + assert_ready_err!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); } #[test] fn close_semaphore_notifies_permit2() { let s = Semaphore::new(2); - let mut permit1 = Permit::new(); - let mut permit2 = Permit::new(); - let mut permit3 = Permit::new(); - let mut permit4 = Permit::new(); + let mut permit1 = task::spawn(Permit::new()); + let mut permit2 = task::spawn(Permit::new()); + let mut permit3 = task::spawn(Permit::new()); + let mut permit4 = task::spawn(Permit::new()); // Acquire a couple of permits - assert_ready_ok!(task::spawn(permit1.acquire(1, &s)).poll()); - assert_ready_ok!(task::spawn(permit2.acquire(1, &s)).poll()); + assert_ready_ok!(permit1.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit2.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); - let mut acquire3 = task::spawn(permit3.acquire(1, &s)); - let mut acquire4 = task::spawn(permit4.acquire(1, &s)); - assert_pending!(acquire3.poll()); - assert_pending!(acquire4.poll()); + assert_pending!(permit3.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_pending!(permit4.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); s.close(); - assert!(acquire3.is_woken()); - assert!(acquire4.is_woken()); + assert!(permit3.is_woken()); + assert!(permit4.is_woken()); - assert_ready_err!(acquire3.poll()); - assert_ready_err!(acquire4.poll()); + assert_ready_err!(permit3.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_ready_err!(permit4.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); assert_eq!(0, s.available_permits()); @@ -331,150 +321,150 @@ fn close_semaphore_notifies_permit2() { assert_eq!(1, s.available_permits()); - assert_ready_err!(task::spawn(permit1.acquire(1, &s)).poll()); + assert_ready_err!(permit1.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); permit2.release(1, &s); assert_eq!(2, s.available_permits()); } -// #[test] -// fn poll_acquire_additional_permits_while_waiting_before_assigned() { -// let s = Semaphore::new(1); +#[test] +fn poll_acquire_additional_permits_while_waiting_before_assigned() { + let s = Semaphore::new(1); -// let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = task::spawn(Permit::new()); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 3, &s))); -// s.add_permits(1); -// assert!(!permit.is_woken()); + s.add_permits(1); + assert!(!permit.is_woken()); -// s.add_permits(1); -// assert!(permit.is_woken()); + s.add_permits(1); + assert!(permit.is_woken()); -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); -// } + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 3, &s))); +} -// #[test] -// fn try_acquire_additional_permits_while_waiting_before_assigned() { -// let s = Semaphore::new(1); +#[test] +fn try_acquire_additional_permits_while_waiting_before_assigned() { + let s = Semaphore::new(1); -// let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = task::spawn(Permit::new()); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); -// assert_err!(permit.enter(|_, mut p| p.try_acquire(3, &s))); + assert_err!(permit.enter(|_, mut p| p.try_acquire(3, &s))); -// s.add_permits(1); -// assert!(permit.is_woken()); + s.add_permits(1); + assert!(permit.is_woken()); -// assert_ok!(permit.enter(|_, mut p| p.try_acquire(2, &s))); -// } + assert_ok!(permit.enter(|_, mut p| p.try_acquire(2, &s))); +} -// #[test] -// fn poll_acquire_additional_permits_while_waiting_after_assigned_success() { -// let s = Semaphore::new(1); +#[test] +fn poll_acquire_additional_permits_while_waiting_after_assigned_success() { + let s = Semaphore::new(1); -// let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = task::spawn(Permit::new()); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); -// s.add_permits(2); + s.add_permits(2); -// assert!(permit.is_woken()); -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); -// } + assert!(permit.is_woken()); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 3, &s))); +} -// #[test] -// fn poll_acquire_additional_permits_while_waiting_after_assigned_requeue() { -// let s = Semaphore::new(1); +#[test] +fn poll_acquire_additional_permits_while_waiting_after_assigned_requeue() { + let s = Semaphore::new(1); -// let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = task::spawn(Permit::new()); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); -// s.add_permits(2); + s.add_permits(2); -// assert!(permit.is_woken()); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); + assert!(permit.is_woken()); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 4, &s))); -// s.add_permits(1); + s.add_permits(1); -// assert!(permit.is_woken()); -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); -// } + assert!(permit.is_woken()); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 4, &s))); +} -// #[test] -// fn poll_acquire_fewer_permits_while_waiting() { -// let s = Semaphore::new(1); +#[test] +fn poll_acquire_fewer_permits_while_waiting() { + let s = Semaphore::new(1); -// let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = task::spawn(Permit::new()); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// assert_eq!(s.available_permits(), 0); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_eq!(s.available_permits(), 0); -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -// assert_eq!(s.available_permits(), 0); -// } + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); + assert_eq!(s.available_permits(), 0); +} -// #[test] -// fn poll_acquire_fewer_permits_after_assigned() { -// let s = Semaphore::new(1); +#[test] +fn poll_acquire_fewer_permits_after_assigned() { + let s = Semaphore::new(1); -// let mut permit1 = task::spawn(Box::new(Permit::new())); -// let mut permit2 = task::spawn(Box::new(Permit::new())); + let mut permit1 = task::spawn(Permit::new()); + let mut permit2 = task::spawn(Permit::new()); -// assert_pending!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); -// assert_eq!(s.available_permits(), 0); + assert_pending!(permit1.enter(|cx, mut p| p.poll_acquire(cx, 5, &s))); + assert_eq!(s.available_permits(), 0); -// assert_pending!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_pending!(permit2.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); -// s.add_permits(4); -// assert!(permit1.is_woken()); -// assert!(!permit2.is_woken()); + s.add_permits(4); + assert!(permit1.is_woken()); + assert!(!permit2.is_woken()); -// assert_ready_ok!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); + assert_ready_ok!(permit1.enter(|cx, mut p| p.poll_acquire(cx, 3, &s))); -// assert!(permit2.is_woken()); -// assert_eq!(s.available_permits(), 1); + assert!(permit2.is_woken()); + assert_eq!(s.available_permits(), 1); -// assert_ready_ok!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -// } + assert_ready_ok!(permit2.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); +} -// #[test] -// fn forget_partial_1() { -// let s = Semaphore::new(0); +#[test] +fn forget_partial_1() { + let s = Semaphore::new(0); -// let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = task::spawn(Permit::new()); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// s.add_permits(1); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + s.add_permits(1); -// assert_eq!(0, s.available_permits()); + assert_eq!(0, s.available_permits()); -// permit.release(1, &s); + permit.release(1, &s); -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 1, &s))); -// assert_eq!(s.available_permits(), 0); -// } + assert_eq!(s.available_permits(), 0); +} -// #[test] -// fn forget_partial_2() { -// let s = Semaphore::new(0); +#[test] +fn forget_partial_2() { + let s = Semaphore::new(0); -// let mut permit = task::spawn(Box::new(Permit::new())); + let mut permit = task::spawn(Permit::new()); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// s.add_permits(1); + assert_pending!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + s.add_permits(1); -// assert_eq!(0, s.available_permits()); + assert_eq!(0, s.available_permits()); -// permit.release(1, &s); + permit.release(1, &s); -// s.add_permits(1); + s.add_permits(1); -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// assert_eq!(s.available_permits(), 0); -// } + assert_ready_ok!(permit.enter(|cx, mut p| p.poll_acquire(cx, 2, &s))); + assert_eq!(s.available_permits(), 0); +} From 1337d8e5499c5f5e95f8e7cee2f1ac7b5c8f6196 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 6 Mar 2020 11:47:41 -0800 Subject: [PATCH 12/91] add loom tests Signed-off-by: Eliza Weisman --- tokio/src/sync/tests/loom_semaphore_batch.rs | 170 +++++++++++++++++++ tokio/src/sync/tests/mod.rs | 1 + 2 files changed, 171 insertions(+) create mode 100644 tokio/src/sync/tests/loom_semaphore_batch.rs diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs new file mode 100644 index 00000000000..33601712c4a --- /dev/null +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -0,0 +1,170 @@ +use crate::sync::batch_semaphore::*; + +use futures::future::poll_fn; +use loom::future::block_on; +use loom::thread; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering::SeqCst; +use std::sync::Arc; +use std::task::Poll::Ready; +use std::task::{Context, Poll}; + +#[test] +fn basic_usage() { + const NUM: usize = 2; + + struct Shared { + semaphore: Semaphore, + active: AtomicUsize, + } + + async fn actor(shared: Arc) { + let mut permit = Permit::new(); + permit.acquire(1, &shared.semaphore).await.unwrap(); + + let actual = shared.active.fetch_add(1, SeqCst); + assert!(actual <= NUM - 1); + + let actual = shared.active.fetch_sub(1, SeqCst); + assert!(actual <= NUM); + + permit.release(1, &shared.semaphore); + } + + loom::model(|| { + let shared = Arc::new(Shared { + semaphore: Semaphore::new(NUM), + active: AtomicUsize::new(0), + }); + + for _ in 0..NUM { + let shared = shared.clone(); + + thread::spawn(move || { + block_on(actor(shared)); + }); + } + + block_on(actor(shared)); + }); +} + +#[test] +fn release() { + loom::model(|| { + let semaphore = Arc::new(Semaphore::new(1)); + + { + let semaphore = semaphore.clone(); + thread::spawn(move || { + let mut permit = Permit::new(); + + block_on(permit.acquire(1, &semaphore)).unwrap(); + + permit.release(1, &semaphore); + }); + } + + let mut permit = Permit::new(); + + block_on(permit.acquire(1, &semaphore)).unwrap(); + + permit.release(1, &semaphore); + }); +} + +#[test] +fn basic_closing() { + const NUM: usize = 2; + + loom::model(|| { + let semaphore = Arc::new(Semaphore::new(1)); + + for _ in 0..NUM { + let semaphore = semaphore.clone(); + + thread::spawn(move || { + let mut permit = Permit::new(); + + for _ in 0..2 { + block_on(permit.acquire(1, &semaphore)).map_err(|_|())?; + + permit.release(1, &semaphore); + } + + Ok::<(), ()>(()) + }); + } + + semaphore.close(); + }); +} + +#[test] +fn concurrent_close() { + const NUM: usize = 3; + + loom::model(|| { + let semaphore = Arc::new(Semaphore::new(1)); + + for _ in 0..NUM { + let semaphore = semaphore.clone(); + + thread::spawn(move || { + let mut permit = Permit::new(); + + block_on(permit.acquire(1, &semaphore)).map_err(|_|())?; + + permit.release(1, &semaphore); + + semaphore.close(); + + Ok::<(), ()>(()) + }); + } + }); +} + +#[test] +fn batch() { + let mut b = loom::model::Builder::new(); + b.preemption_bound = Some(1); + + b.check(|| { + let semaphore = Arc::new(Semaphore::new(10)); + let active = Arc::new(AtomicUsize::new(0)); + let mut ths = vec![]; + + for _ in 0..2 { + let semaphore = semaphore.clone(); + let active = active.clone(); + + ths.push(thread::spawn(move || { + let mut permit = Permit::new(); + + for n in &[4, 10, 8] { + block_on(permit.acquire(*n, &semaphore)).unwrap(); + + active.fetch_add(*n as usize, SeqCst); + + let num_active = active.load(SeqCst); + assert!(num_active <= 10); + + thread::yield_now(); + + active.fetch_sub(*n as usize, SeqCst); + + permit.release(*n, &semaphore); + } + })); + } + + for th in ths.into_iter() { + th.join().unwrap(); + } + + assert_eq!(10, semaphore.available_permits()); + }); +} diff --git a/tokio/src/sync/tests/mod.rs b/tokio/src/sync/tests/mod.rs index 32926e49375..d571754c011 100644 --- a/tokio/src/sync/tests/mod.rs +++ b/tokio/src/sync/tests/mod.rs @@ -11,5 +11,6 @@ cfg_loom! { mod loom_mpsc; mod loom_notify; mod loom_oneshot; + mod loom_semaphore_batch; mod loom_semaphore_ll; } From 19e0157e80217fdad2ddbbcd97ae4272e941a0d1 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 6 Mar 2020 15:01:16 -0800 Subject: [PATCH 13/91] decr assigned permits when assigning to self Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 40 ++++++++++++++++++------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 26488faf268..3c8e0633853 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -17,6 +17,11 @@ use std::{ }, }; +macro_rules! ddbg { + ($x:expr) => { $x }; + ($($x:expr),+) => {} +} + pub(crate) struct Semaphore { waiters: Mutex>, permits: AtomicUsize, @@ -103,7 +108,7 @@ impl Semaphore { /// Adds `n` new permits to the semaphore. pub(crate) fn add_permits(&self, n: usize) { - dbg!(n); + ddbg!(n); if n == 0 { return; } @@ -112,7 +117,7 @@ impl Semaphore { // abort. let prev = self.add_lock.fetch_add(n << 1, Ordering::AcqRel); - let closed = match dbg!(prev) { + let closed = match ddbg!(prev) { 1 => true, 0 => false, // Another thread has the lock and will be responsible for notifying @@ -120,7 +125,7 @@ impl Semaphore { _ => return, }; - // if self.permits.load(Ordering::Acquire) & CLOSED == 1 { + // if self.permits.load(Ordering::Acquire) & CLOgpg*SED == 1 { // } @@ -138,11 +143,12 @@ impl Semaphore { let initial = rem; // Release the permits and notify - while dbg!(rem > 0) || dbg!(closed) { + while ddbg!(rem > 0) || ddbg!(closed) { let pop = match waiters.last() { - Some(last) => dbg!(last.assign_permits(&mut rem, closed)), + Some(last) => ddbg!(last.assign_permits(&mut rem, closed)), None => { self.permits.fetch_add(rem, Ordering::Release); + rem = 0; break; // false } @@ -157,15 +163,15 @@ impl Semaphore { let actual = if closed { let actual = self.add_lock.fetch_sub(n | 1, Ordering::AcqRel); closed = false; - actual + actual >> 1 } else { let actual = self.add_lock.fetch_sub(n, Ordering::AcqRel); closed = actual & 1 == 1; - actual + actual >> 1 }; - rem = actual.saturating_sub(initial) >> 1; - dbg!(rem); + rem = (actual >> 1) - rem; + ddbg!(rem); } } @@ -178,12 +184,12 @@ impl Semaphore { let mut curr = self.permits.load(Ordering::Acquire); let needed = needed as usize; let (acquired, remaining) = loop { - dbg!(needed, curr); - if dbg!(curr & CLOSED == CLOSED) { + ddbg!(needed, curr); + if ddbg!(curr & CLOSED == CLOSED) { return Ready(Err(AcquireError(()))); } let mut remaining = 0; - let (next, acquired) = if dbg!(curr) >= dbg!(needed) { + let (next, acquired) = if ddbg!(curr) >= ddbg!(needed) { (curr - needed, needed) } else { remaining = needed - curr; @@ -199,7 +205,7 @@ impl Semaphore { Err(actual) => curr = actual, } }; - dbg!(acquired, remaining); + ddbg!(acquired, remaining); if remaining == 0 { return Ready(Ok(())); } @@ -334,7 +340,7 @@ impl Permit { Acquired(acquired) => { let n = cmp::min(n, acquired); self.state = Acquired(acquired - n); - dbg!(n) + ddbg!(n) } } } @@ -430,7 +436,7 @@ impl Future for Acquire<'_> { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let (node, semaphore, permit, mut needed) = self.project(); - dbg!(&semaphore, &permit, &needed); + ddbg!(&semaphore, &permit, &needed); permit.state = match permit.state { PermitState::Acquired(n) if n >= needed => return Ready(Ok(())), PermitState::Acquired(n) => { @@ -472,8 +478,8 @@ impl Acquire<'_> { impl Drop for Acquire<'_> { fn drop(&mut self) { - dbg!("drop acquire"); - if dbg!(self.node.is_unlinked()) { + ddbg!("drop acquire"); + if ddbg!(self.node.is_unlinked()) { // don't need to release permits return; } From 2764ba276cdfc033f121090144d9def1096324df Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 6 Mar 2020 16:50:03 -0800 Subject: [PATCH 14/91] closed means REALLY closed --- tokio/src/sync/batch_semaphore.rs | 36 +++++++++++++------- tokio/src/sync/tests/loom_semaphore_batch.rs | 1 + 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 3c8e0633853..9140bfc70be 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -23,11 +23,16 @@ macro_rules! ddbg { } pub(crate) struct Semaphore { - waiters: Mutex>, + waiters: Mutex, permits: AtomicUsize, add_lock: AtomicUsize, } +struct Waitlist { + queue: LinkedList, + closed: bool, +} + /// Error returned by `Permit::try_acquire`. #[derive(Debug)] pub(crate) enum TryAcquireError { @@ -80,7 +85,10 @@ impl Semaphore { assert!(permits <= std::u16::MAX as usize); Self { permits: AtomicUsize::new(permits), - waiters: Mutex::new(LinkedList::new()), + waiters: Mutex::new(Waitlist { + queue: LinkedList::new(), + closed: false, + }), add_lock: AtomicUsize::new(0), } } @@ -95,7 +103,7 @@ impl Semaphore { pub(crate) fn close(&self) { // Acquire the `add_lock`, setting the "closed" flag on the lock. let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); - self.permits.fetch_or(CLOSED, Ordering::Release); + println!("closed"); if prev != 0 { // Another thread has the lock and will be responsible for notifying @@ -103,7 +111,9 @@ impl Semaphore { return; } - self.add_permits_locked(0, true, self.waiters.lock().unwrap()); + let mut lock = self.waiters.lock().unwrap(); + lock.closed = true; + self.add_permits_locked(0, true, lock); } /// Adds `n` new permits to the semaphore. @@ -136,7 +146,7 @@ impl Semaphore { &self, mut rem: usize, mut closed: bool, - mut waiters: MutexGuard<'_, LinkedList>, + mut waiters: MutexGuard<'_, Waitlist>, ) { while rem > 0 || closed { // how many permits are we releasing on this pass? @@ -144,7 +154,7 @@ impl Semaphore { let initial = rem; // Release the permits and notify while ddbg!(rem > 0) || ddbg!(closed) { - let pop = match waiters.last() { + let pop = match waiters.queue.last() { Some(last) => ddbg!(last.assign_permits(&mut rem, closed)), None => { self.permits.fetch_add(rem, Ordering::Release); @@ -154,7 +164,7 @@ impl Semaphore { } }; if pop { - waiters.pop_back().unwrap(); + waiters.queue.pop_back().unwrap(); } } @@ -185,9 +195,6 @@ impl Semaphore { let needed = needed as usize; let (acquired, remaining) = loop { ddbg!(needed, curr); - if ddbg!(curr & CLOSED == CLOSED) { - return Ready(Err(AcquireError(()))); - } let mut remaining = 0; let (next, acquired) = if ddbg!(curr) >= ddbg!(needed) { (curr - needed, needed) @@ -222,11 +229,14 @@ impl Semaphore { // otherwise, register the waker & enqueue the node. node.waker.register_by_ref(cx.waker()); - let mut queue = self.waiters.lock().unwrap(); + let mut waiters = self.waiters.lock().unwrap(); + if waiters.closed { + return Ready(Err(AcquireError(()))); + } unsafe { // XXX(eliza) T_T let node = Pin::into_inner_unchecked(node) as *mut _; - queue.push_front(node); + waiters.queue.push_front(node); println!("enqueue"); } @@ -492,7 +502,7 @@ impl Drop for Acquire<'_> { // // safety: the waiter is only added to `waiters` by virtue of it // being the only `LinkedList` available to the type. - unsafe { waiters.remove(NonNull::from(&mut self.node)) }; + unsafe { waiters.queue.remove(NonNull::from(&mut self.node)) }; // TODO(eliza): release permits to next waiter } diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index 33601712c4a..e5a0f394c8e 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -80,6 +80,7 @@ fn basic_closing() { const NUM: usize = 2; loom::model(|| { + println!("-- iter --"); let semaphore = Arc::new(Semaphore::new(1)); for _ in 0..NUM { From d5ed4400f87e6e0ad1d7f28f4ee1e655d88707c5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 9 Mar 2020 15:29:55 -0700 Subject: [PATCH 15/91] broken Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 140 ++++++++++++------- tokio/src/sync/tests/loom_semaphore_batch.rs | 2 + 2 files changed, 92 insertions(+), 50 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 9140bfc70be..ac5bef1642a 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -18,7 +18,7 @@ use std::{ }; macro_rules! ddbg { - ($x:expr) => { $x }; + ($x:expr) => { dbg!($x) }; ($($x:expr),+) => {} } @@ -105,58 +105,63 @@ impl Semaphore { let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); println!("closed"); - if prev != 0 { + if dbg!(prev) != 0 { // Another thread has the lock and will be responsible for notifying // pending waiters. return; } - let mut lock = self.waiters.lock().unwrap(); lock.closed = true; - self.add_permits_locked(0, true, lock); + println!("locked"); + self.add_permits_locked(0, lock, true); } /// Adds `n` new permits to the semaphore. - pub(crate) fn add_permits(&self, n: usize) { - ddbg!(n); - if n == 0 { + pub(crate) fn add_permits(&self, added: usize) { + ddbg!(added); + if added == 0 { return; } // TODO: Handle overflow. A panic is not sufficient, the process must // abort. - let prev = self.add_lock.fetch_add(n << 1, Ordering::AcqRel); - - let closed = match ddbg!(prev) { - 1 => true, - 0 => false, - // Another thread has the lock and will be responsible for notifying - // pending waiters. - _ => return, - }; + let prev = self.add_lock.fetch_add(added << 1, Ordering::AcqRel); + if prev > 0 { + return + } + // let closed = match ddbg!(prev) { + // 1 => true, + // 0 => false, + // // Another thread has the lock and will be responsible for notifying + // // pending waiters. + // _ => return, + // }; // if self.permits.load(Ordering::Acquire) & CLOgpg*SED == 1 { // } - self.add_permits_locked(n, false, self.waiters.lock().unwrap()); + self.add_permits_locked(added, self.waiters.lock().unwrap(), false); } fn add_permits_locked( &self, mut rem: usize, - mut closed: bool, mut waiters: MutexGuard<'_, Waitlist>, + mut closed: bool, ) { - while rem > 0 || closed { - // how many permits are we releasing on this pass? + println!(" ADD PERMITS LOCKED "); + + loop { + // how many permits are we releasing on this pass? let initial = rem; // Release the permits and notify - while ddbg!(rem > 0) || ddbg!(closed) { + while dbg!(rem) > 0 || dbg!(waiters.closed) { let pop = match waiters.queue.last() { - Some(last) => ddbg!(last.assign_permits(&mut rem, closed)), + Some(last) => ddbg!(last.assign_permits(&mut rem, waiters.closed)), None => { + println!("queue empty"); self.permits.fetch_add(rem, Ordering::Release); rem = 0; break; @@ -168,21 +173,33 @@ impl Semaphore { } } - let n = (initial - rem) << 1; + let n = (dbg!(initial).saturating_sub(dbg!(rem))) << 1; + + dbg!(n); let actual = if closed { - let actual = self.add_lock.fetch_sub(n | 1, Ordering::AcqRel); + let actual = dbg!(self.add_lock.fetch_sub(n | 1, Ordering::AcqRel)); + assert!(actual < std::usize::MAX); closed = false; actual >> 1 } else { - let actual = self.add_lock.fetch_sub(n, Ordering::AcqRel); - closed = actual & 1 == 1; + let actual = dbg!(self.add_lock.fetch_sub(n, Ordering::AcqRel)); + assert!(actual < std::usize::MAX); + if actual & 1 == 1 { + waiters.closed = true; + closed = true; + } actual >> 1 }; - rem = (actual >> 1) - rem; - ddbg!(rem); + rem = dbg!(actual.saturating_sub(n >> 1)); + + if dbg!(rem) == 0 && !dbg!(closed) { + break; + } } + + println!("DONE ADDING PERMITS"); } fn poll_acquire( @@ -191,6 +208,7 @@ impl Semaphore { needed: u16, node: Pin<&mut Waiter>, ) -> Poll> { + loop { let mut curr = self.permits.load(Ordering::Acquire); let needed = needed as usize; let (acquired, remaining) = loop { @@ -208,6 +226,7 @@ impl Semaphore { Ordering::AcqRel, Ordering::Acquire, ) { + Ok(_) => break (acquired, remaining), Err(actual) => curr = actual, } @@ -217,32 +236,53 @@ impl Semaphore { return Ready(Ok(())); } - assert!(node.is_unlinked()); - node.state - .compare_exchange( - Waiter::UNQUEUED, - remaining, - Ordering::Release, - Ordering::Relaxed, - ) - .expect("not unqueued"); - // otherwise, register the waker & enqueue the node. - node.waker.register_by_ref(cx.waker()); - - let mut waiters = self.waiters.lock().unwrap(); - if waiters.closed { + let mut waiters = if let Ok(lock) = self.waiters.try_lock() { + lock + } else { + // "fence" to wait until permits are not being added and retry + let _ = self.waiters.lock().unwrap(); + continue; + }; + if dbg!(waiters.closed) { return Ready(Err(AcquireError(()))); } - unsafe { - // XXX(eliza) T_T - let node = Pin::into_inner_unchecked(node) as *mut _; - waiters.queue.push_front(node); - println!("enqueue"); + + let mut state = dbg!(node.state.load(Ordering::Acquire)); + let mut next = state; + loop { + // were we assigned permits while blocking on the lock? + next = if dbg!(state >= Waiter::UNQUEUED) { + dbg!(remaining) + } else { + dbg!(state.saturating_sub(acquired)) + }; + match node.state.compare_exchange(state, next, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => break, + Err(actual) => state = actual, + } + + } + + if next & std::u16::MAX as usize == 0 { + return Ready(Ok(())); + } + if node.is_unlinked() { + // otherwise, register the waker & enqueue the node. + node.waker.register_by_ref(cx.waker()); + + unsafe { + // XXX(eliza) T_T + let node = Pin::into_inner_unchecked(node) as *mut _; + waiters.queue.push_front(node); + println!("enqueue"); + } } - Pending + + return Pending } } +} impl Drop for Semaphore { fn drop(&mut self) { @@ -381,7 +421,7 @@ impl Waiter { /// /// Returns `true` if the waiter should be removed from the queue fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { - if closed { + if dbg!(closed) { self.waker.wake(); return true; } @@ -398,7 +438,7 @@ impl Waiter { { Ok(_) => { // Update `n` - *n -= assign; + *n = n.saturating_sub(assign); if next == 0 { if curr > 0 { diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index e5a0f394c8e..1a87abf201d 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -34,6 +34,8 @@ fn basic_usage() { } loom::model(|| { + + println!("\n ------ iter ------ \n"); let shared = Arc::new(Shared { semaphore: Semaphore::new(NUM), active: AtomicUsize::new(0), From cb472fd9b0440a0fb38a38417935288e8c70c49c Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 9 Mar 2020 16:09:11 -0700 Subject: [PATCH 16/91] wip broken Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 12 ++++++++++-- tokio/src/sync/tests/loom_semaphore_batch.rs | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index ac5bef1642a..3253f8da51c 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -159,7 +159,13 @@ impl Semaphore { // Release the permits and notify while dbg!(rem) > 0 || dbg!(waiters.closed) { let pop = match waiters.queue.last() { - Some(last) => ddbg!(last.assign_permits(&mut rem, waiters.closed)), + Some(last) => { + + ddbg!(format_args!("assign permits to {:p}", last)); + let res = ddbg!(last.assign_permits(&mut rem, waiters.closed)); + dbg!(last.is_unlinked()); + res + } None => { println!("queue empty"); self.permits.fetch_add(rem, Ordering::Release); @@ -169,7 +175,9 @@ impl Semaphore { } }; if pop { + println!("popping"); waiters.queue.pop_back().unwrap(); + println!("popped"); } } @@ -528,7 +536,7 @@ impl Acquire<'_> { impl Drop for Acquire<'_> { fn drop(&mut self) { - ddbg!("drop acquire"); + ddbg!(format_args!("drop acquire {:p}", self)); if ddbg!(self.node.is_unlinked()) { // don't need to release permits return; diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index 1a87abf201d..e93b8233f67 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -23,7 +23,7 @@ fn basic_usage() { async fn actor(shared: Arc) { let mut permit = Permit::new(); permit.acquire(1, &shared.semaphore).await.unwrap(); - + println!("acquired!"); let actual = shared.active.fetch_add(1, SeqCst); assert!(actual <= NUM - 1); From bd1c5dba34135d3fb5081b36f1ef2ad1b777fc48 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 9 Mar 2020 16:15:03 -0700 Subject: [PATCH 17/91] update after merge Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 3253f8da51c..769fd597f38 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -281,7 +281,7 @@ impl Semaphore { unsafe { // XXX(eliza) T_T let node = Pin::into_inner_unchecked(node) as *mut _; - waiters.queue.push_front(node); + waiters.queue.push_front(NonNull::new_unchecked(node)); println!("enqueue"); } } @@ -612,16 +612,15 @@ impl std::error::Error for TryAcquireError {} /// /// `Waiter` is forced to be !Unpin. unsafe impl linked_list::Link for Waiter { - type Handle = *mut Waiter; + type Handle = NonNull; type Target = Waiter; - fn to_raw(handle: *mut Waiter) -> NonNull { - debug_assert!(!handle.is_null()); - unsafe { NonNull::new_unchecked(handle) } + fn as_raw(handle: &Self::Handle) -> NonNull { + *handle } - unsafe fn from_raw(ptr: NonNull) -> *mut Waiter { - ptr.as_ptr() + unsafe fn from_raw(ptr: NonNull) -> NonNull { + ptr } unsafe fn pointers(mut target: NonNull) -> NonNull> { From 1880121d87f9d61a8f5973ad14d43d199b6c952e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 10 Mar 2020 12:33:45 -0700 Subject: [PATCH 18/91] almost evherything works Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 197 ++++++++++--------- tokio/src/sync/tests/loom_semaphore_batch.rs | 2 +- tokio/src/util/linked_list.rs | 28 +++ 3 files changed, 131 insertions(+), 96 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 769fd597f38..bf1b191edbd 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -18,8 +18,10 @@ use std::{ }; macro_rules! ddbg { - ($x:expr) => { dbg!($x) }; - ($($x:expr),+) => {} + ($x:expr) => { + dbg!($x) + }; + ($($x:expr),+) => {}; } pub(crate) struct Semaphore { @@ -85,7 +87,7 @@ impl Semaphore { assert!(permits <= std::u16::MAX as usize); Self { permits: AtomicUsize::new(permits), - waiters: Mutex::new(Waitlist { + waiters: Mutex::new(Waitlist { queue: LinkedList::new(), closed: false, }), @@ -127,7 +129,7 @@ impl Semaphore { // abort. let prev = self.add_lock.fetch_add(added << 1, Ordering::AcqRel); if prev > 0 { - return + return; } // let closed = match ddbg!(prev) { // 1 => true, @@ -150,7 +152,6 @@ impl Semaphore { mut waiters: MutexGuard<'_, Waitlist>, mut closed: bool, ) { - println!(" ADD PERMITS LOCKED "); loop { @@ -159,30 +160,33 @@ impl Semaphore { // Release the permits and notify while dbg!(rem) > 0 || dbg!(waiters.closed) { let pop = match waiters.queue.last() { + Some(_) if waiters.closed => true, Some(last) => { - - ddbg!(format_args!("assign permits to {:p}", last)); + ddbg!(format_args!("assign permits to {:p}", last)); let res = ddbg!(last.assign_permits(&mut rem, waiters.closed)); - dbg!(last.is_unlinked()); + // dbg!(last.is_unlinked()); res } None => { - println!("queue empty"); - self.permits.fetch_add(rem, Ordering::Release); - rem = 0; + dbg!("queue empty"); + let _prev = self.permits.fetch_add(rem, Ordering::Release); + dbg!(_prev + rem); break; // false } }; if pop { - println!("popping"); - waiters.queue.pop_back().unwrap(); - println!("popped"); + dbg!("popping"); + let waiter = waiters.queue.pop_back().unwrap(); + dbg!(format_args!("popped {:?}", waiter)); + unsafe { + waiter.as_ref().waker.wake(); + } + dbg!("woke"); } } - - let n = (dbg!(initial).saturating_sub(dbg!(rem))) << 1; + let n = initial << 1; dbg!(n); let actual = if closed { @@ -200,7 +204,7 @@ impl Semaphore { actual >> 1 }; - rem = dbg!(actual.saturating_sub(n >> 1)); + rem = actual - initial; if dbg!(rem) == 0 && !dbg!(closed) { break; @@ -216,81 +220,92 @@ impl Semaphore { needed: u16, node: Pin<&mut Waiter>, ) -> Poll> { + let mut lock = None; + let mut acquired = 0; + let mut should_push = false; loop { - let mut curr = self.permits.load(Ordering::Acquire); - let needed = needed as usize; - let (acquired, remaining) = loop { - ddbg!(needed, curr); - let mut remaining = 0; - let (next, acquired) = if ddbg!(curr) >= ddbg!(needed) { - (curr - needed, needed) - } else { - remaining = needed - curr; - (0, curr) + let mut curr = self.permits.load(Ordering::Acquire); + let needed = needed as usize; + let remaining = loop { + ddbg!(needed, curr); + let mut remaining = 0; + let (next, acq) = if ddbg!(curr + acquired) >= ddbg!(needed) { + let next = curr - (needed - acquired); + (dbg!(next), needed) + } else { + remaining = (needed - acquired) - curr; + (0, curr) + }; + match self.permits.compare_exchange_weak( + curr, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + acquired += acq; + break remaining; + } + Err(actual) => curr = actual, + } }; - match self.permits.compare_exchange_weak( - curr, - next, - Ordering::AcqRel, - Ordering::Acquire, - ) { - - Ok(_) => break (acquired, remaining), - Err(actual) => curr = actual, + ddbg!(acquired, remaining); + if remaining == 0 { + return Ready(Ok(())); } - }; - ddbg!(acquired, remaining); - if remaining == 0 { - return Ready(Ok(())); - } - let mut waiters = if let Ok(lock) = self.waiters.try_lock() { - lock - } else { - // "fence" to wait until permits are not being added and retry - let _ = self.waiters.lock().unwrap(); - continue; - }; - if dbg!(waiters.closed) { - return Ready(Err(AcquireError(()))); - } - - let mut state = dbg!(node.state.load(Ordering::Acquire)); - let mut next = state; - loop { - // were we assigned permits while blocking on the lock? - next = if dbg!(state >= Waiter::UNQUEUED) { - dbg!(remaining) - } else { - dbg!(state.saturating_sub(acquired)) + let mut waiters = match lock { + Some(lock) => lock, + None => { + lock = Some(self.waiters.lock().unwrap()); + continue; + } }; - match node.state.compare_exchange(state, next, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => break, - Err(actual) => state = actual, + if dbg!(waiters.closed) { + return Ready(Err(AcquireError(()))); + } + + let mut state = dbg!(node.state.load(Ordering::Acquire)); + let mut next = state; + loop { + // were we assigned permits while blocking on the lock? + next = if dbg!(state >= Waiter::UNQUEUED) { + should_push = true; + dbg!(remaining) + } else { + dbg!(state.saturating_sub(acquired)) + }; + match node + .state + .compare_exchange(state, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => break, + Err(actual) => state = actual, + } + } + + if next & std::u16::MAX as usize == 0 { + return Ready(Ok(())); } - } - - if next & std::u16::MAX as usize == 0 { - return Ready(Ok(())); - } - if node.is_unlinked() { // otherwise, register the waker & enqueue the node. node.waker.register_by_ref(cx.waker()); unsafe { // XXX(eliza) T_T let node = Pin::into_inner_unchecked(node) as *mut _; - waiters.queue.push_front(NonNull::new_unchecked(node)); - println!("enqueue"); + let node = NonNull::new_unchecked(node); + + if !waiters.queue.is_linked(&node) { + waiters.queue.push_front(node); + println!("enqueue"); + } } - } - - return Pending + return Pending; + } } } -} impl Drop for Semaphore { fn drop(&mut self) { @@ -421,19 +436,18 @@ impl Waiter { } } - fn is_unlinked(&self) -> bool { - self.pointers.is_unlinked() - } - /// Assign permits to the waiter. /// /// Returns `true` if the waiter should be removed from the queue fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { if dbg!(closed) { - self.waker.wake(); return true; } let mut curr = self.state.load(Ordering::Acquire); + dbg!(format_args!( + "assigning {} permits to {:p} (curr: {})", + n, self, curr + )); loop { // Number of permits to assign to this waiter @@ -448,15 +462,7 @@ impl Waiter { // Update `n` *n = n.saturating_sub(assign); - if next == 0 { - if curr > 0 { - self.waker.wake(); - } - - return true; - } else { - return false; - } + return next == 0; } Err(actual) => curr = actual, } @@ -537,20 +543,21 @@ impl Acquire<'_> { impl Drop for Acquire<'_> { fn drop(&mut self) { ddbg!(format_args!("drop acquire {:p}", self)); - if ddbg!(self.node.is_unlinked()) { - // don't need to release permits - return; - } + // This is where we ensure safety. The future is being dropped, // which means we must ensure that the waiter entry is no longer stored // in the linked list. let mut waiters = self.semaphore.waiters.lock().unwrap(); - + let node = NonNull::from(&mut self.node); + if ddbg!(!waiters.queue.is_linked(&node)) { + // don't need to release permits + return; + } // remove the entry from the list // // safety: the waiter is only added to `waiters` by virtue of it // being the only `LinkedList` available to the type. - unsafe { waiters.queue.remove(NonNull::from(&mut self.node)) }; + unsafe { waiters.queue.remove(node) }; // TODO(eliza): release permits to next waiter } diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index e93b8233f67..397ecc59e0a 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -5,7 +5,7 @@ use loom::future::block_on; use loom::thread; use std::future::Future; use std::pin::Pin; -use std::sync::atomic::AtomicUsize; +use loom::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::SeqCst; use std::sync::Arc; use std::task::Poll::Ready; diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index cbdde8e132c..88a8b1cc8df 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -83,6 +83,16 @@ impl LinkedList { let ptr = T::as_raw(&*val); unsafe { + dbg!(format_args!( + "ptr={:p}; head={:p}; tail={:p}", + ptr, + self.head + .map(NonNull::as_ptr) + .unwrap_or_else(std::ptr::null_mut), + self.tail + .map(NonNull::as_ptr) + .unwrap_or_else(std::ptr::null_mut) + )); T::pointers(ptr).as_mut().next = self.head; T::pointers(ptr).as_mut().prev = None; @@ -132,6 +142,24 @@ impl LinkedList { true } + pub(crate) fn is_linked(&self, node: &T::Handle) -> bool { + let node = T::as_raw(node); + unsafe { + if T::pointers(node).as_ref().next.is_some() + || T::pointers(node).as_ref().prev.is_some() + { + return true; + } + } + if let Some(head) = self.head { + if head == node { + assert_eq!(self.tail, Some(node)); + return true; + } + } + false + } + /// Removes the specified node from the list /// /// # Safety From c7c08ffd02d1185a53a99fd4e3120cc0173d18d9 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 10 Mar 2020 15:28:35 -0700 Subject: [PATCH 19/91] wip Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 108 +++++++++++++++++++----------- tokio/src/util/linked_list.rs | 12 +--- 2 files changed, 70 insertions(+), 50 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index bf1b191edbd..f82f023e844 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -1,4 +1,5 @@ use crate::loom::{ + cell::CausalCell, future::AtomicWaker, sync::{atomic::AtomicUsize, Mutex, MutexGuard}, }; @@ -103,6 +104,7 @@ impl Semaphore { /// Closes the semaphore. This prevents the semaphore from issuing new /// permits and notifies all pending waiters. pub(crate) fn close(&self) { + self.permits.fetch_or(CLOSED, Ordering::Release); // Acquire the `add_lock`, setting the "closed" flag on the lock. let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); println!("closed"); @@ -169,7 +171,7 @@ impl Semaphore { } None => { dbg!("queue empty"); - let _prev = self.permits.fetch_add(rem, Ordering::Release); + let _prev = self.permits.fetch_add(rem, Ordering::AcqRel); dbg!(_prev + rem); break; // false @@ -220,11 +222,20 @@ impl Semaphore { needed: u16, node: Pin<&mut Waiter>, ) -> Poll> { + let mut state = dbg!(node.state.load(Ordering::Acquire)); + if state == 0 { + return Ready(Ok(())); + } + let mut lock = None; let mut acquired = 0; - let mut should_push = false; - loop { + let mut waiters = loop { let mut curr = self.permits.load(Ordering::Acquire); + + if dbg!(curr & CLOSED > 0) { + return Ready(Err(AcquireError(()))); + } + let needed = needed as usize; let remaining = loop { ddbg!(needed, curr); @@ -254,56 +265,75 @@ impl Semaphore { return Ready(Ok(())); } - let mut waiters = match lock { - Some(lock) => lock, + // No permits were immediately available, so this permit will (probably) need to wait. + // We'll need to acquire a lock on the wait queue. + match lock { None => { + // We haven't acquired the lock yet. If it isn't contended, we can continue. + if let Ok(lock) = self.waiters.try_lock() { + break lock; + } + // Otherwise, the wait list is currently locked. In that case, we'll need to + // check the available permits a second time before we enqueue the node to wait + // --- someone else might be in the process of releasing permits. lock = Some(self.waiters.lock().unwrap()); continue; } + Some(lock) => break lock, }; - if dbg!(waiters.closed) { - return Ready(Err(AcquireError(()))); - } + }; - let mut state = dbg!(node.state.load(Ordering::Acquire)); - let mut next = state; - loop { - // were we assigned permits while blocking on the lock? - next = if dbg!(state >= Waiter::UNQUEUED) { - should_push = true; - dbg!(remaining) - } else { - dbg!(state.saturating_sub(acquired)) - }; - match node - .state - .compare_exchange(state, next, Ordering::AcqRel, Ordering::Acquire) - { - Ok(_) => break, - Err(actual) => state = actual, - } + if dbg!(waiters.closed) { + return Ready(Err(AcquireError(()))); + } + + let mut next = state; + let mut leftover = 0; + loop { + leftover = 0; + // were we assigned permits while blocking on the lock? + next = if dbg!(state >= Waiter::UNQUEUED) { + dbg!(needed as usize - acquired) + } else if acquired > state { + leftover = dbg!(acquired - state); + 0 + } else { + state - acquired + }; + match node + .state + .compare_exchange(state, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => break, + Err(actual) => state = actual, } + } - if next & std::u16::MAX as usize == 0 { - return Ready(Ok(())); + if next & std::u16::MAX as usize == 0 { + // we may have been assigned "bonus permits", and we have to give them back! + drop(waiters); + if leftover > 0 { + self.add_permits(leftover); } - // otherwise, register the waker & enqueue the node. - node.waker.register_by_ref(cx.waker()); + return Ready(Ok(())); + } - unsafe { - // XXX(eliza) T_T - let node = Pin::into_inner_unchecked(node) as *mut _; - let node = NonNull::new_unchecked(node); + // otherwise, register the waker & enqueue the node. + node.waker.register_by_ref(cx.waker()); - if !waiters.queue.is_linked(&node) { - waiters.queue.push_front(node); - println!("enqueue"); - } - } + unsafe { + // XXX(eliza) T_T + let node = Pin::into_inner_unchecked(node) as *mut _; + let node = NonNull::new_unchecked(node); - return Pending; + if !waiters.queue.is_linked(&node) { + waiters.queue.push_front(node); + println!("enqueue"); + } } + + Pending } } diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 88a8b1cc8df..0babe5da3f5 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -81,18 +81,8 @@ impl LinkedList { // The value should not be dropped, it is being inserted into the list let val = ManuallyDrop::new(val); let ptr = T::as_raw(&*val); - + assert_ne!(self.head, Some(ptr)); unsafe { - dbg!(format_args!( - "ptr={:p}; head={:p}; tail={:p}", - ptr, - self.head - .map(NonNull::as_ptr) - .unwrap_or_else(std::ptr::null_mut), - self.tail - .map(NonNull::as_ptr) - .unwrap_or_else(std::ptr::null_mut) - )); T::pointers(ptr).as_mut().next = self.head; T::pointers(ptr).as_mut().prev = None; From 1a4f733dd0dfe08009b0056b756451f7df802106 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 11 Mar 2020 12:19:26 -0700 Subject: [PATCH 20/91] ALL LOOM TESTS PASS Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 108 ++++++++++++++---------------- 1 file changed, 52 insertions(+), 56 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index f82f023e844..d10cb3c748b 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -104,10 +104,11 @@ impl Semaphore { /// Closes the semaphore. This prevents the semaphore from issuing new /// permits and notifies all pending waiters. pub(crate) fn close(&self) { + dbg!("closing..."); self.permits.fetch_or(CLOSED, Ordering::Release); // Acquire the `add_lock`, setting the "closed" flag on the lock. let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); - println!("closed"); + dbg!("closed"); if dbg!(prev) != 0 { // Another thread has the lock and will be responsible for notifying @@ -222,67 +223,58 @@ impl Semaphore { needed: u16, node: Pin<&mut Waiter>, ) -> Poll> { - let mut state = dbg!(node.state.load(Ordering::Acquire)); - if state == 0 { - return Ready(Ok(())); - } - - let mut lock = None; + let mut state; let mut acquired = 0; + let mut lock = None; + let mut curr = self.permits.load(Ordering::Acquire); let mut waiters = loop { - let mut curr = self.permits.load(Ordering::Acquire); + state = dbg!(node.state.load(Ordering::Acquire)); + if state == 0 { + return Ready(Ok(())); + } if dbg!(curr & CLOSED > 0) { return Ready(Err(AcquireError(()))); } - let needed = needed as usize; - let remaining = loop { - ddbg!(needed, curr); - let mut remaining = 0; - let (next, acq) = if ddbg!(curr + acquired) >= ddbg!(needed) { - let next = curr - (needed - acquired); - (dbg!(next), needed) - } else { - remaining = (needed - acquired) - curr; - (0, curr) - }; - match self.permits.compare_exchange_weak( - curr, - next, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => { - acquired += acq; - break remaining; - } - Err(actual) => curr = actual, - } + let needed = cmp::min(state, needed as usize); + ddbg!(needed, curr); + let mut remaining = 0; + let (next, acq) = if ddbg!(curr + acquired) >= ddbg!(needed) { + let next = curr - (needed - acquired); + (dbg!(next), needed) + } else { + remaining = (needed - acquired) - curr; + (0, curr) }; - ddbg!(acquired, remaining); - if remaining == 0 { - return Ready(Ok(())); + if remaining > 0 && lock.is_none() { + // No permits were immediately available, so this permit will (probably) need to wait. + // We'll need to acquire a lock on the wait queue. + // Otherwise, the wait list is currently locked. In that case, we'll need to + // check the available permits a second time before we enqueue the node to wait + // --- someone else might be in the process of releasing permits. + lock = Some(self.waiters.lock().unwrap()); + dbg!("LOCK ACQUIRED"); + continue; } - - // No permits were immediately available, so this permit will (probably) need to wait. - // We'll need to acquire a lock on the wait queue. - match lock { - None => { - // We haven't acquired the lock yet. If it isn't contended, we can continue. - if let Ok(lock) = self.waiters.try_lock() { - break lock; + match self + .permits + .compare_exchange_weak(curr, next, Ordering::AcqRel, Ordering::AcqRel) + { + Ok(_) => { + ddbg!(acquired, remaining); + acquired += acq; + if remaining == 0 { + return Ready(Ok(())); } - // Otherwise, the wait list is currently locked. In that case, we'll need to - // check the available permits a second time before we enqueue the node to wait - // --- someone else might be in the process of releasing permits. - lock = Some(self.waiters.lock().unwrap()); - continue; + break lock; } - Some(lock) => break lock, - }; + Err(actual) => curr = actual, + } }; + let mut waiters = waiters.unwrap(); + dbg!("LOCKED"); if dbg!(waiters.closed) { return Ready(Err(AcquireError(()))); } @@ -300,21 +292,24 @@ impl Semaphore { } else { state - acquired }; - match node - .state - .compare_exchange(state, next, Ordering::AcqRel, Ordering::Acquire) - { + match node.state.compare_exchange( + dbg!(state), + dbg!(next), + Ordering::AcqRel, + Ordering::Acquire, + ) { Ok(_) => break, Err(actual) => state = actual, } } + if leftover > 0 { + panic!("unexpected bonus permits {:?}", leftover); + } + if next & std::u16::MAX as usize == 0 { // we may have been assigned "bonus permits", and we have to give them back! drop(waiters); - if leftover > 0 { - self.add_permits(leftover); - } return Ready(Ok(())); } @@ -401,6 +396,7 @@ impl Permit { /// Releases a permit back to the semaphore pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { + dbg!("release", n); let n = self.forget(n, semaphore); semaphore.add_permits(n as usize); } From a320d56862009da17fa6cac955ffc04848063fb3 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 11 Mar 2020 14:11:22 -0700 Subject: [PATCH 21/91] fix typoed fail ordering Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index d10cb3c748b..8a52d55a952 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -257,10 +257,12 @@ impl Semaphore { dbg!("LOCK ACQUIRED"); continue; } - match self - .permits - .compare_exchange_weak(curr, next, Ordering::AcqRel, Ordering::AcqRel) - { + match self.permits.compare_exchange_weak( + curr, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { Ok(_) => { ddbg!(acquired, remaining); acquired += acq; From df1e7bb01739ca831a658695e83f0ee4033c046c Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 11 Mar 2020 15:46:59 -0700 Subject: [PATCH 22/91] remove some warnings Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 8a52d55a952..3ab75717ac9 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -1,5 +1,4 @@ use crate::loom::{ - cell::CausalCell, future::AtomicWaker, sync::{atomic::AtomicUsize, Mutex, MutexGuard}, }; @@ -227,7 +226,7 @@ impl Semaphore { let mut acquired = 0; let mut lock = None; let mut curr = self.permits.load(Ordering::Acquire); - let mut waiters = loop { + let waiters = loop { state = dbg!(node.state.load(Ordering::Acquire)); if state == 0 { return Ready(Ok(())); @@ -281,10 +280,8 @@ impl Semaphore { return Ready(Err(AcquireError(()))); } - let mut next = state; - let mut leftover = 0; + let mut next; loop { - leftover = 0; // were we assigned permits while blocking on the lock? next = if dbg!(state >= Waiter::UNQUEUED) { dbg!(needed as usize - acquired) @@ -305,14 +302,7 @@ impl Semaphore { } } - if leftover > 0 { - panic!("unexpected bonus permits {:?}", leftover); - } - if next & std::u16::MAX as usize == 0 { - // we may have been assigned "bonus permits", and we have to give them back! - drop(waiters); - return Ready(Ok(())); } @@ -326,7 +316,6 @@ impl Semaphore { if !waiters.queue.is_linked(&node) { waiters.queue.push_front(node); - println!("enqueue"); } } @@ -398,7 +387,6 @@ impl Permit { /// Releases a permit back to the semaphore pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { - dbg!("release", n); let n = self.forget(n, semaphore); semaphore.add_permits(n as usize); } From b957af8bfb1d4fbc211b69740bc65855e9635ddd Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 10:55:23 -0700 Subject: [PATCH 23/91] bring back try_acquire Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 87 ++++++++++++++++++------------- 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 3ab75717ac9..207ab9abdf1 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -133,17 +133,6 @@ impl Semaphore { if prev > 0 { return; } - // let closed = match ddbg!(prev) { - // 1 => true, - // 0 => false, - // // Another thread has the lock and will be responsible for notifying - // // pending waiters. - // _ => return, - // }; - - // if self.permits.load(Ordering::Acquire) & CLOgpg*SED == 1 { - - // } self.add_permits_locked(added, self.waiters.lock().unwrap(), false); } @@ -216,6 +205,31 @@ impl Semaphore { println!("DONE ADDING PERMITS"); } + fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { + let mut curr = self.permits.load(Ordering::Acquire); + loop { + // Has the semaphore closed? + if curr & CLOSED > 0 { + return Err(TryAcquireError::Closed); + } + + // Are there enough permits remaining? + if (curr as u16) < num_permits { + return Err(TryAcquireError::NoPermits); + } + + let next = curr - num_permits as usize; + + match self + .permits + .compare_exchange(curr, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return Ok(()), + Err(actual) => curr = actual, + } + } + } + fn poll_acquire( &self, cx: &mut Context<'_>, @@ -224,15 +238,20 @@ impl Semaphore { ) -> Poll> { let mut state; let mut acquired = 0; + + // First, try to take the requested number of permits from the semaphore. let mut lock = None; let mut curr = self.permits.load(Ordering::Acquire); let waiters = loop { state = dbg!(node.state.load(Ordering::Acquire)); + + // Has the waiter already acquired all its needed permits? If so, we're done! if state == 0 { return Ready(Ok(())); } - if dbg!(curr & CLOSED > 0) { + // Has the semaphore closed? + if curr & CLOSED > 0 { return Ready(Err(AcquireError(()))); } @@ -246,6 +265,7 @@ impl Semaphore { remaining = (needed - acquired) - curr; (0, curr) }; + if remaining > 0 && lock.is_none() { // No permits were immediately available, so this permit will (probably) need to wait. // We'll need to acquire a lock on the wait queue. @@ -253,9 +273,9 @@ impl Semaphore { // check the available permits a second time before we enqueue the node to wait // --- someone else might be in the process of releasing permits. lock = Some(self.waiters.lock().unwrap()); - dbg!("LOCK ACQUIRED"); continue; } + match self.permits.compare_exchange_weak( curr, next, @@ -286,7 +306,6 @@ impl Semaphore { next = if dbg!(state >= Waiter::UNQUEUED) { dbg!(needed as usize - acquired) } else if acquired > state { - leftover = dbg!(acquired - state); 0 } else { state - acquired @@ -382,7 +401,21 @@ impl Permit { num_permits: u16, semaphore: &Semaphore, ) -> Result<(), TryAcquireError> { - unimplemented!() + use PermitState::*; + + match self.state { + Waiting(_) => unreachable!( + "if a permit is waiting, then it has been borrowed mutably by an `Acquire` future" + ), + Acquired(acquired) => { + if acquired < num_permits { + semaphore.try_acquire(num_permits - acquired)?; + self.state = Acquired(num_permits); + } + } + } + + Ok(()) } /// Releases a permit back to the semaphore @@ -405,27 +438,9 @@ impl Permit { use PermitState::*; match self.state { - Waiting(requested) => { - panic!( - "cannot forget permits while in wait queue; we are already borrowed mutably?" - ) - // let n = cmp::min(n, requested); - // let node = unsafe { &*self.node.get() }; - // // Decrement - // let acquired = node.try_dec_permits_to_acquire(n as usize) as u16; - - // if n == requested { - // self.state = Acquired(0); - // // // TODO: rm from wait list here! - // // semaphore - // } else if acquired == requested - n { - // self.state = Waiting(acquired); - // } else { - // self.state = Waiting(requested - n); - // } - - // acquired - } + Waiting(requested) => panic!( + "cannot forget permits while in wait queue; we are already borrowed mutably?" + ), Acquired(acquired) => { let n = cmp::min(n, acquired); self.state = Acquired(acquired - n); From ec2b4e73213dc19149158f1af30a6ed8209bd6f7 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 10:58:05 -0700 Subject: [PATCH 24/91] misc cleanup Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 8 ++++---- tokio/src/sync/semaphore.rs | 2 +- tokio/src/sync/tests/semaphore_batch.rs | 2 +- tokio/src/util/linked_list.rs | 5 ----- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 207ab9abdf1..215e50be2d3 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -420,7 +420,7 @@ impl Permit { /// Releases a permit back to the semaphore pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { - let n = self.forget(n, semaphore); + let n = self.forget(n); semaphore.add_permits(n as usize); } @@ -434,11 +434,11 @@ impl Permit { /// /// Will forget **at most** the number of acquired permits. This number is /// returned. - pub(crate) fn forget(&mut self, n: u16, semaphore: &Semaphore) -> u16 { + pub(crate) fn forget(&mut self, n: u16) -> u16 { use PermitState::*; match self.state { - Waiting(requested) => panic!( + Waiting(_) => unreachable!( "cannot forget permits while in wait queue; we are already borrowed mutably?" ), Acquired(acquired) => { @@ -530,7 +530,7 @@ impl Future for Acquire<'_> { type Output = Result<(), AcquireError>; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let (node, semaphore, permit, mut needed) = self.project(); + let (node, semaphore, permit, needed) = self.project(); ddbg!(&semaphore, &permit, &needed); permit.state = match permit.state { PermitState::Acquired(n) if n >= needed => return Ready(Ok(())), diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index 3216ce1cd9f..c1e8ba7787f 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -83,7 +83,7 @@ impl<'a> SemaphorePermit<'a> { /// This can be used to reduce the amount of permits available from a /// semaphore. pub fn forget(mut self) { - self.ll_permit.forget(1, &self.sem.ll_sem); + self.ll_permit.forget(1); } } diff --git a/tokio/src/sync/tests/semaphore_batch.rs b/tokio/src/sync/tests/semaphore_batch.rs index 776cf1aa4d3..630aaf97795 100644 --- a/tokio/src/sync/tests/semaphore_batch.rs +++ b/tokio/src/sync/tests/semaphore_batch.rs @@ -132,7 +132,7 @@ fn forget_acquired() { assert_eq!(s.available_permits(), 0); - permit.forget(1, &s); + permit.forget(1); assert_eq!(s.available_permits(), 0); } diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 0babe5da3f5..5faf3d37df9 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -230,11 +230,6 @@ impl Pointers { next: None, } } - - /// Returns `true` if this set of pointers is not part of a list. - pub(crate) fn is_unlinked(&self) -> bool { - self.prev.is_none() && self.next.is_none() - } } #[cfg(test)] From 770d8596cad0f71940f2f7c1c9b57290137204ca Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 11:20:24 -0700 Subject: [PATCH 25/91] add test for release of leftover permits Signed-off-by: Eliza Weisman --- tokio/src/sync/tests/loom_semaphore_batch.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index 397ecc59e0a..fe13ae16857 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -171,3 +171,23 @@ fn batch() { assert_eq!(10, semaphore.available_permits()); }); } + +#[test] +fn release_during_acquire() { + loom::model(|| { + let semaphore = Arc::new(Semaphore::new(10)); + let mut permit1 = Permit::new(); + permit1.try_acquire(8, &semaphore).expect("try_acquire should succeed; semaphore uncontended"); + let semaphore2 = semaphore.clone(); + let thread = thread::spawn(move || { + let mut permit = Permit::new(); + block_on(permit.acquire(4, &semaphore2)).unwrap(); + permit + }); + + permit1.release(8, &semaphore); + let mut permit2 = thread.join().unwrap(); + permit2.release(4, &semaphore); + assert_eq!(10, semaphore.available_permits()); + }) +} \ No newline at end of file From 38f654f067894e3e502febe5c4a4d24040a2314b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 11:53:43 -0700 Subject: [PATCH 26/91] simplify `poll_acquire` behavior Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 72 ++++++++++--------------------- 1 file changed, 23 insertions(+), 49 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 215e50be2d3..5d56d3fbd72 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -252,7 +252,7 @@ impl Semaphore { // Has the semaphore closed? if curr & CLOSED > 0 { - return Ready(Err(AcquireError(()))); + return Ready(Err(AcquireError::closed())); } let needed = cmp::min(state, needed as usize); @@ -269,10 +269,10 @@ impl Semaphore { if remaining > 0 && lock.is_none() { // No permits were immediately available, so this permit will (probably) need to wait. // We'll need to acquire a lock on the wait queue. - // Otherwise, the wait list is currently locked. In that case, we'll need to - // check the available permits a second time before we enqueue the node to wait - // --- someone else might be in the process of releasing permits. lock = Some(self.waiters.lock().unwrap()); + // While we were waiting to lock the wait list, additional permits may have been + // released. Therefore, we will acquire a new snapshot of the current state of the + // semaphore before actually enqueueing the waiter.. continue; } @@ -300,28 +300,27 @@ impl Semaphore { return Ready(Err(AcquireError(()))); } - let mut next; - loop { - // were we assigned permits while blocking on the lock? - next = if dbg!(state >= Waiter::UNQUEUED) { - dbg!(needed as usize - acquired) - } else if acquired > state { - 0 - } else { - state - acquired - }; - match node.state.compare_exchange( - dbg!(state), - dbg!(next), - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => break, - Err(actual) => state = actual, + let next = match acquired.cmp(&state) { + // if the waiter is in the unqueued state, we need all the requested permits, minus the amount we just acquired. + _ if state >= Waiter::UNQUEUED => needed as usize - acquired, + // We have acquired all the needed permits! + cmp::Ordering::Equal => 0, + cmp::Ordering::Less => state - acquired, + cmp::Ordering::Greater => { + unreachable!("cannot have been assigned more than needed permits") } - } + }; + + // Typically, one would probably expect to see a compare-and-swap loop here. However, in + // this case, we don't need to loop. Our most snapshot of the node's current state + // was acquired after we acquired the lock on the wait list. Because releasing permits to + // waiters also requires locking the wait list, the state cannot have changed since we + // acquired this snapshot. + node.state + .compare_exchange(state, next, Ordering::AcqRel, Ordering::Acquire) + .expect("state cannot have changed while the wait list was locked"); - if next & std::u16::MAX as usize == 0 { + if next == 0 { return Ready(Ok(())); } @@ -499,31 +498,6 @@ impl Waiter { } } } - - /// Try to decrement the number of permits to acquire. This returns the - /// actual number of permits that were decremented. The delta betweeen `n` - /// and the return has been assigned to the permit and the caller must - /// assign these back to the semaphore. - fn try_dec_permits_to_acquire(&self, n: usize) -> usize { - let mut curr = self.state.load(Ordering::Acquire); - - loop { - // if !curr.is_queued() { - // assert_eq!(0, curr.permits_to_acquire()); - // } - - let delta = cmp::min(n, curr); - let rem = curr - delta; - - match self - .state - .compare_exchange(curr, rem, Ordering::AcqRel, Ordering::Acquire) - { - Ok(_) => return n - delta, - Err(actual) => curr = actual, - } - } - } } impl Future for Acquire<'_> { From 9046b042f6616ecf2e009c3df442d2489637f9fd Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 12:06:33 -0700 Subject: [PATCH 27/91] rm unneeded AtomicWaker Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 5d56d3fbd72..9c1d226612f 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -1,5 +1,5 @@ use crate::loom::{ - future::AtomicWaker, + cell::CausalCell, sync::{atomic::AtomicUsize, Mutex, MutexGuard}, }; use crate::util::linked_list::{self, LinkedList}; @@ -14,6 +14,7 @@ use std::{ task::{ Context, Poll, Poll::{Pending, Ready}, + Waker, }, }; @@ -69,7 +70,7 @@ enum PermitState { } struct Waiter { - waker: AtomicWaker, + waker: CausalCell>, state: AtomicUsize, /// Intrusive linked-list pointers @@ -168,11 +169,14 @@ impl Semaphore { }; if pop { dbg!("popping"); - let waiter = waiters.queue.pop_back().unwrap(); + let mut waiter = waiters.queue.pop_back().unwrap(); dbg!(format_args!("popped {:?}", waiter)); - unsafe { - waiter.as_ref().waker.wake(); - } + let waker = unsafe { + waiter.as_mut().waker.with_mut(|waker| { + (*waker).take() + }) + }; + waker.expect("if a node is in the wait list, it must have a waker").wake(); dbg!("woke"); } } @@ -325,7 +329,10 @@ impl Semaphore { } // otherwise, register the waker & enqueue the node. - node.waker.register_by_ref(cx.waker()); + node.waker.with_mut(|waker| + // safety: the wait list is locked, so we may modify the waker. + unsafe { *waker = Some(cx.waker().clone()) } + ); unsafe { // XXX(eliza) T_T @@ -459,7 +466,7 @@ impl Waiter { const UNQUEUED: usize = 1 << 16; fn new() -> Self { Waiter { - waker: AtomicWaker::new(), + waker: CausalCell::new(None), state: AtomicUsize::new(Self::UNQUEUED), pointers: linked_list::Pointers::new(), _p: PhantomPinned, From 187e4d7d77148d081a9e2992b86a38efaf2ad8b5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 12:09:58 -0700 Subject: [PATCH 28/91] remove unused imports Signed-off-by: Eliza Weisman --- tokio/src/sync/mutex.rs | 2 -- tokio/src/sync/rwlock.rs | 3 --- tokio/src/sync/semaphore.rs | 2 -- 3 files changed, 7 deletions(-) diff --git a/tokio/src/sync/mutex.rs b/tokio/src/sync/mutex.rs index 39f33402baa..1af9eb0e6b1 100644 --- a/tokio/src/sync/mutex.rs +++ b/tokio/src/sync/mutex.rs @@ -78,8 +78,6 @@ //! //! [`Mutex`]: struct.Mutex.html //! [`MutexGuard`]: struct.MutexGuard.html - -use crate::future::poll_fn; use crate::sync::batch_semaphore as semaphore; use std::cell::UnsafeCell; diff --git a/tokio/src/sync/rwlock.rs b/tokio/src/sync/rwlock.rs index 5992ab47f16..d9d049ea953 100644 --- a/tokio/src/sync/rwlock.rs +++ b/tokio/src/sync/rwlock.rs @@ -1,9 +1,6 @@ -use crate::future::poll_fn; use crate::sync::batch_semaphore::{AcquireError, Permit, Semaphore}; use std::cell::UnsafeCell; use std::ops; -use std::pin::Pin; -use std::task::{Context, Poll}; #[cfg(not(loom))] const MAX_READS: usize = 32; diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index c1e8ba7787f..5566fa79942 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -1,6 +1,4 @@ use super::batch_semaphore as ll; // low level implementation -use crate::future::poll_fn; -use std::pin::Pin; /// Counting semaphore performing asynchronous permit aquisition. /// From c2450a241448064e120d68eb9b9770332f121bb4 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 12:20:06 -0700 Subject: [PATCH 29/91] add test for cancelling an acquire future Signed-off-by: Eliza Weisman --- tokio/src/sync/tests/semaphore_batch.rs | 86 ++++++++++++++++--------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/tokio/src/sync/tests/semaphore_batch.rs b/tokio/src/sync/tests/semaphore_batch.rs index 630aaf97795..7fea0032cc8 100644 --- a/tokio/src/sync/tests/semaphore_batch.rs +++ b/tokio/src/sync/tests/semaphore_batch.rs @@ -197,52 +197,52 @@ fn poll_acquire_many_unavailable() { assert_ready_ok!(acquire_3.poll()); } -// #[test] -// fn try_acquire_one_unavailable() { -// let s = Semaphore::new(1); +#[test] +fn try_acquire_one_unavailable() { + let s = Semaphore::new(1); -// let mut permit_1 = Permit::new(); -// let mut permit_2 = Permit::new(); + let mut permit_1 = Permit::new(); + let mut permit_2 = Permit::new(); -// // Acquire the first permit -// assert_ok!(permit_1.try_acquire(1, &s)); -// assert_eq!(s.available_permits(), 0); + // Acquire the first permit + assert_ok!(permit_1.try_acquire(1, &s)); + assert_eq!(s.available_permits(), 0); -// assert_err!(permit_2.try_acquire(1, &s)); + assert_err!(permit_2.try_acquire(1, &s)); -// permit_1.release(1, &s); + permit_1.release(1, &s); -// assert_eq!(s.available_permits(), 1); -// assert_ok!(permit_2.try_acquire(1, &s)); + assert_eq!(s.available_permits(), 1); + assert_ok!(permit_2.try_acquire(1, &s)); -// permit_2.release(1, &s); -// assert_eq!(s.available_permits(), 1); -// } + permit_2.release(1, &s); + assert_eq!(s.available_permits(), 1); +} -// #[test] -// fn try_acquire_many_unavailable() { -// let s = Semaphore::new(5); +#[test] +fn try_acquire_many_unavailable() { + let s = Semaphore::new(5); -// let mut permit_1 = Permit::new(); -// let mut permit_2 = Permit::new(); + let mut permit_1 = Permit::new(); + let mut permit_2 = Permit::new(); -// // Acquire the first permit -// assert_ok!(permit_1.try_acquire(1, &s)); -// assert_eq!(s.available_permits(), 4); + // Acquire the first permit + assert_ok!(permit_1.try_acquire(1, &s)); + assert_eq!(s.available_permits(), 4); -// assert_err!(permit_2.try_acquire(5, &s)); + assert_err!(permit_2.try_acquire(5, &s)); -// permit_1.release(1, &s); -// assert_eq!(s.available_permits(), 5); + permit_1.release(1, &s); + assert_eq!(s.available_permits(), 5); -// assert_ok!(permit_2.try_acquire(5, &s)); + assert_ok!(permit_2.try_acquire(5, &s)); -// permit_2.release(1, &s); -// assert_eq!(s.available_permits(), 1); + permit_2.release(1, &s); + assert_eq!(s.available_permits(), 1); -// permit_2.release(1, &s); -// assert_eq!(s.available_permits(), 2); -// } + permit_2.release(1, &s); + assert_eq!(s.available_permits(), 2); +} #[test] fn poll_acquire_one_zero_permits() { @@ -478,3 +478,25 @@ fn close_semaphore_notifies_permit2() { // assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); // assert_eq!(s.available_permits(), 0); // } + +#[test] +fn cancel_acquire_releases_permits() { + let s = Semaphore::new(10); + let mut permit1 = Permit::new(); + permit1 + .try_acquire(4, &s) + .expect("uncontended try_acquire succeeds"); + assert_eq!(6, s.available_permits()); + + let mut permit2 = Permit::new(); + let mut acquire = task::spawn(permit2.acquire(8, &s)); + assert_pending!(acquire.poll()); + + assert_eq!(0, s.available_permits()); + drop(acquire); + + assert_eq!(6, s.available_permits()); + permit2 + .try_acquire(6, &s) + .expect("should acquire successfully"); +} From ddc1726202d627af7272b9b9ca97426266d69082 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 12:44:44 -0700 Subject: [PATCH 30/91] dropping an incomplete `Acquire` releases permits Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 9c1d226612f..606bd09b086 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -561,17 +561,27 @@ impl Drop for Acquire<'_> { // in the linked list. let mut waiters = self.semaphore.waiters.lock().unwrap(); let node = NonNull::from(&mut self.node); - if ddbg!(!waiters.queue.is_linked(&node)) { - // don't need to release permits - return; - } - // remove the entry from the list - // - // safety: the waiter is only added to `waiters` by virtue of it - // being the only `LinkedList` available to the type. - unsafe { waiters.queue.remove(node) }; + if waiters.queue.is_linked(&node) { + // remove the entry from the list + // + // safety: the waiter is only added to `waiters` by virtue of it + // being the only `LinkedList` available to the type. + unsafe { waiters.queue.remove(node) }; + + let acquired_permits = self.num_permits as usize - self.node.state.load(Ordering::Acquire); + // we have already locked the mutex, so we know we will be the one to release these + // permits, but it's necessary to add them since we will try to subtract them once we have + // finished permit assignment. + self.semaphore.add_lock.fetch_add(acquired_permits << 1, Ordering::AcqRel); + self.semaphore.add_permits_locked(acquired_permits, waiters, false); + } else { + drop(waiters); + }; - // TODO(eliza): release permits to next waiter + // If the permit had transitioned to the `Waiting` state, put it back into `Acquired`. + if let PermitState::Waiting(_) = self.permit.state { + self.permit.state = PermitState::Acquired(0); + } } } From 1aa808887c30082648dee2953115360c0c9f79bd Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 12:53:28 -0700 Subject: [PATCH 31/91] no need to lock if unqueued Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 606bd09b086..d49cd72ffe3 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -556,6 +556,11 @@ impl Drop for Acquire<'_> { fn drop(&mut self) { ddbg!(format_args!("drop acquire {:p}", self)); + // fast path: if we aren't actually waiting, no need to acquire the lock. + if self.node.state.load(Ordering::Acquire) & Waiter::UNQUEUED == Waiter::UNQUEUED { + return; + } + // This is where we ensure safety. The future is being dropped, // which means we must ensure that the waiter entry is no longer stored // in the linked list. From 88b5e86406c0d70a00a1f62ed8b9f95d990ca648 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 12:59:49 -0700 Subject: [PATCH 32/91] remove dbg/println Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 35 ++++++++----------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index d49cd72ffe3..9e0546ac2de 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -104,26 +104,23 @@ impl Semaphore { /// Closes the semaphore. This prevents the semaphore from issuing new /// permits and notifies all pending waiters. pub(crate) fn close(&self) { - dbg!("closing..."); self.permits.fetch_or(CLOSED, Ordering::Release); // Acquire the `add_lock`, setting the "closed" flag on the lock. let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); - dbg!("closed"); - if dbg!(prev) != 0 { + if prev != 0 { // Another thread has the lock and will be responsible for notifying // pending waiters. return; } + let mut lock = self.waiters.lock().unwrap(); lock.closed = true; - println!("locked"); self.add_permits_locked(0, lock, true); } /// Adds `n` new permits to the semaphore. pub(crate) fn add_permits(&self, added: usize) { - ddbg!(added); if added == 0 { return; } @@ -144,8 +141,6 @@ impl Semaphore { mut waiters: MutexGuard<'_, Waitlist>, mut closed: bool, ) { - println!(" ADD PERMITS LOCKED "); - loop { // how many permits are we releasing on this pass? let initial = rem; @@ -153,31 +148,20 @@ impl Semaphore { while dbg!(rem) > 0 || dbg!(waiters.closed) { let pop = match waiters.queue.last() { Some(_) if waiters.closed => true, - Some(last) => { - ddbg!(format_args!("assign permits to {:p}", last)); - let res = ddbg!(last.assign_permits(&mut rem, waiters.closed)); - // dbg!(last.is_unlinked()); - res - } + Some(last) => last.assign_permits(&mut rem, waiters.closed), None => { - dbg!("queue empty"); - let _prev = self.permits.fetch_add(rem, Ordering::AcqRel); - dbg!(_prev + rem); + self.permits.fetch_add(rem, Ordering::AcqRel); break; - // false } }; if pop { - dbg!("popping"); let mut waiter = waiters.queue.pop_back().unwrap(); - dbg!(format_args!("popped {:?}", waiter)); let waker = unsafe { waiter.as_mut().waker.with_mut(|waker| { (*waker).take() }) }; waker.expect("if a node is in the wait list, it must have a waker").wake(); - dbg!("woke"); } } @@ -185,13 +169,13 @@ impl Semaphore { dbg!(n); let actual = if closed { - let actual = dbg!(self.add_lock.fetch_sub(n | 1, Ordering::AcqRel)); - assert!(actual < std::usize::MAX); + let actual = self.add_lock.fetch_sub(n | 1, Ordering::AcqRel); + assert!(actual <= CLOSED); closed = false; actual >> 1 } else { - let actual = dbg!(self.add_lock.fetch_sub(n, Ordering::AcqRel)); - assert!(actual < std::usize::MAX); + let actual = self.add_lock.fetch_sub(n, Ordering::AcqRel); + assert!(actual <= CLOSED); if actual & 1 == 1 { waiters.closed = true; closed = true; @@ -201,12 +185,11 @@ impl Semaphore { rem = actual - initial; - if dbg!(rem) == 0 && !dbg!(closed) { + if rem == 0 && !closed { break; } } - println!("DONE ADDING PERMITS"); } fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { From cf27256b0da5199036fde61d739047ef6ddfe2a2 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 13:00:08 -0700 Subject: [PATCH 33/91] remove more dbg/printlns Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 9e0546ac2de..23b191d7364 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -229,8 +229,8 @@ impl Semaphore { // First, try to take the requested number of permits from the semaphore. let mut lock = None; let mut curr = self.permits.load(Ordering::Acquire); - let waiters = loop { - state = dbg!(node.state.load(Ordering::Acquire)); + let mut waiters = loop { + state = node.state.load(Ordering::Acquire); // Has the waiter already acquired all its needed permits? If so, we're done! if state == 0 { @@ -243,11 +243,10 @@ impl Semaphore { } let needed = cmp::min(state, needed as usize); - ddbg!(needed, curr); let mut remaining = 0; - let (next, acq) = if ddbg!(curr + acquired) >= ddbg!(needed) { + let (next, acq) = if curr + acquired >= needed { let next = curr - (needed - acquired); - (dbg!(next), needed) + (next, needed) } else { remaining = (needed - acquired) - curr; (0, curr) @@ -270,20 +269,17 @@ impl Semaphore { Ordering::Acquire, ) { Ok(_) => { - ddbg!(acquired, remaining); acquired += acq; if remaining == 0 { return Ready(Ok(())); } - break lock; + break lock.unwrap(); } Err(actual) => curr = actual, } }; - let mut waiters = waiters.unwrap(); - dbg!("LOCKED"); - if dbg!(waiters.closed) { + if waiters.closed { return Ready(Err(AcquireError(()))); } @@ -460,14 +456,11 @@ impl Waiter { /// /// Returns `true` if the waiter should be removed from the queue fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { - if dbg!(closed) { + if closed { return true; } let mut curr = self.state.load(Ordering::Acquire); - dbg!(format_args!( - "assigning {} permits to {:p} (curr: {})", - n, self, curr - )); + loop { // Number of permits to assign to this waiter From cb73e3f0814fc2d5d4d2a037e2d27fae1ff2251f Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Mar 2020 13:37:25 -0700 Subject: [PATCH 34/91] add comments/remove dbgs Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 23b191d7364..b22bf6fb54a 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -18,13 +18,6 @@ use std::{ }, }; -macro_rules! ddbg { - ($x:expr) => { - dbg!($x) - }; - ($($x:expr),+) => {}; -} - pub(crate) struct Semaphore { waiters: Mutex, permits: AtomicUsize, @@ -145,7 +138,7 @@ impl Semaphore { // how many permits are we releasing on this pass? let initial = rem; // Release the permits and notify - while dbg!(rem) > 0 || dbg!(waiters.closed) { + while rem > 0 || waiters.closed { let pop = match waiters.queue.last() { Some(_) if waiters.closed => true, Some(last) => last.assign_permits(&mut rem, waiters.closed), @@ -167,7 +160,6 @@ impl Semaphore { let n = initial << 1; - dbg!(n); let actual = if closed { let actual = self.add_lock.fetch_sub(n | 1, Ordering::AcqRel); assert!(actual <= CLOSED); @@ -429,7 +421,7 @@ impl Permit { Acquired(acquired) => { let n = cmp::min(n, acquired); self.state = Acquired(acquired - n); - ddbg!(n) + n } } } @@ -456,25 +448,25 @@ impl Waiter { /// /// Returns `true` if the waiter should be removed from the queue fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { + // If the wait list has closed, consume no permits but pop the node. if closed { return true; } - let mut curr = self.state.load(Ordering::Acquire); + let mut curr = self.state.load(Ordering::Acquire); loop { - // Number of permits to assign to this waiter + // Assign up to `n` permits. let assign = cmp::min(curr, *n); let next = curr - assign; - let next = if closed { next | CLOSED } else { next }; match self .state .compare_exchange(curr, next, Ordering::AcqRel, Ordering::Acquire) { Ok(_) => { - // Update `n` *n = n.saturating_sub(assign); + // If we have assigned all remaining permits, return true. return next == 0; } Err(actual) => curr = actual, @@ -488,7 +480,6 @@ impl Future for Acquire<'_> { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let (node, semaphore, permit, needed) = self.project(); - ddbg!(&semaphore, &permit, &needed); permit.state = match permit.state { PermitState::Acquired(n) if n >= needed => return Ready(Ok(())), PermitState::Acquired(n) => { @@ -530,8 +521,6 @@ impl Acquire<'_> { impl Drop for Acquire<'_> { fn drop(&mut self) { - ddbg!(format_args!("drop acquire {:p}", self)); - // fast path: if we aren't actually waiting, no need to acquire the lock. if self.node.state.load(Ordering::Acquire) & Waiter::UNQUEUED == Waiter::UNQUEUED { return; @@ -545,8 +534,7 @@ impl Drop for Acquire<'_> { if waiters.queue.is_linked(&node) { // remove the entry from the list // - // safety: the waiter is only added to `waiters` by virtue of it - // being the only `LinkedList` available to the type. + // safety: we have locked the wait list. unsafe { waiters.queue.remove(node) }; let acquired_permits = self.num_permits as usize - self.node.state.load(Ordering::Acquire); From ce667622c4287fabbf1a7a81b5bfab957db5dc5f Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 13 Mar 2020 11:13:17 -0700 Subject: [PATCH 35/91] add comments/cleanup Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 83 +++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index b22bf6fb54a..5e3af3a5431 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -1,3 +1,19 @@ +//! # Implementation Details +//! +//! The semaphore is implemented using an intrusive linked list of waiters. An atomic counter +//! tracks the number of available permits. If the semaphore does not contain the required number +//! of permits, the task attempting to acquire permits places its waker at the end of a queue. When +//! new permits are made available (such as by releasing an initial acquisition), they are assigned +//! to the task at the front of the queue, waking that task if its requested number of permits is +//! met. +//! +//! Because waiters are enqueued at the back of the linked list and dequeued from the front, the +//! semaphore is fair. Tasks trying to acquire large numbers of permits at a time will always be +//! woken eventually, even if many other tasks are acquiring smaller numbers of permits. This means +//! that in a use-case like tokio's read-write lock, writers will not be starved by readers. +//! +//! The linked list is guarded by a mutex, which must be acquired before enqueueing or dequeueing a +//! task. However, some operations are always wait-free. use crate::loom::{ cell::CausalCell, sync::{atomic::AtomicUsize, Mutex, MutexGuard}, @@ -18,9 +34,17 @@ use std::{ }, }; +/// An asynchronous counting semaphore which permits waiting on multiple permits at once. pub(crate) struct Semaphore { waiters: Mutex, + /// The current number of available permits in the semaphore. permits: AtomicUsize, + /// Permits in the process of being released. + /// + /// When releasing permits, if the lock on the semaphore's wait list is held by another task, + /// the task releasing permits adds them to this counter. The task holding the lock will + /// continue releasing permits until the counter is drained, allowing the `release` operation + /// to be wait free. add_lock: AtomicUsize, } @@ -91,7 +115,7 @@ impl Semaphore { /// Returns the current number of available permits pub(crate) fn available_permits(&self) -> usize { - self.permits.load(Ordering::Acquire) & std::u16::MAX as usize + self.permits.load(Ordering::SeqCst) & std::u16::MAX as usize } /// Closes the semaphore. This prevents the semaphore from issuing new @@ -122,9 +146,13 @@ impl Semaphore { // abort. let prev = self.add_lock.fetch_add(added << 1, Ordering::AcqRel); if prev > 0 { + // Another thread is already assigning permits. It will continue to do so until + // `add_lock` is drained, so we don't need to block on the lock. return; } + // Otherwise, no other thread is assigning permits. Thus, we must lock the semaphore's wait + // list and assign permits until `add_lock` is drained. self.add_permits_locked(added, self.waiters.lock().unwrap(), false); } @@ -148,13 +176,14 @@ impl Semaphore { } }; if pop { - let mut waiter = waiters.queue.pop_back().unwrap(); - let waker = unsafe { - waiter.as_mut().waker.with_mut(|waker| { - (*waker).take() - }) - }; + let node = waiters.queue.pop_back().unwrap(); + let waiter = unsafe { node.as_ref() }; + let waker = waiter.waker.with_mut(|waker| unsafe { + (*waker).take() + }); + waker.expect("if a node is in the wait list, it must have a waker").wake(); + } } @@ -162,12 +191,12 @@ impl Semaphore { let actual = if closed { let actual = self.add_lock.fetch_sub(n | 1, Ordering::AcqRel); - assert!(actual <= CLOSED); + assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); closed = false; actual >> 1 } else { let actual = self.add_lock.fetch_sub(n, Ordering::AcqRel); - assert!(actual <= CLOSED); + assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); if actual & 1 == 1 { waiters.closed = true; closed = true; @@ -181,7 +210,6 @@ impl Semaphore { break; } } - } fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { @@ -223,7 +251,6 @@ impl Semaphore { let mut curr = self.permits.load(Ordering::Acquire); let mut waiters = loop { state = node.state.load(Ordering::Acquire); - // Has the waiter already acquired all its needed permits? If so, we're done! if state == 0 { return Ready(Ok(())); @@ -269,6 +296,7 @@ impl Semaphore { } Err(actual) => curr = actual, } + drop(lock.take()); }; if waiters.closed { @@ -285,10 +313,10 @@ impl Semaphore { unreachable!("cannot have been assigned more than needed permits") } }; - + // Typically, one would probably expect to see a compare-and-swap loop here. However, in // this case, we don't need to loop. Our most snapshot of the node's current state - // was acquired after we acquired the lock on the wait list. Because releasing permits to + // was acquired after we acquired the lock on the wait list. Because releasing permits tPPo // waiters also requires locking the wait list, the state cannot have changed since we // acquired this snapshot. node.state @@ -521,28 +549,32 @@ impl Acquire<'_> { impl Drop for Acquire<'_> { fn drop(&mut self) { + let state = self.node.state.load(Ordering::Acquire); // fast path: if we aren't actually waiting, no need to acquire the lock. - if self.node.state.load(Ordering::Acquire) & Waiter::UNQUEUED == Waiter::UNQUEUED { + if state & Waiter::UNQUEUED == Waiter::UNQUEUED { return; } - // This is where we ensure safety. The future is being dropped, // which means we must ensure that the waiter entry is no longer stored // in the linked list. let mut waiters = self.semaphore.waiters.lock().unwrap(); let node = NonNull::from(&mut self.node); if waiters.queue.is_linked(&node) { + + let acquired_permits = self.num_permits as usize - state; // remove the entry from the list // // safety: we have locked the wait list. unsafe { waiters.queue.remove(node) }; - let acquired_permits = self.num_permits as usize - self.node.state.load(Ordering::Acquire); - // we have already locked the mutex, so we know we will be the one to release these - // permits, but it's necessary to add them since we will try to subtract them once we have - // finished permit assignment. - self.semaphore.add_lock.fetch_add(acquired_permits << 1, Ordering::AcqRel); - self.semaphore.add_permits_locked(acquired_permits, waiters, false); + if acquired_permits > 0 { + // we have already locked the mutex, so we know we will be the one to release these + // permits, but it's necessary to add them since we will try to subtract them once we have + // finished permit assignment. + self.semaphore.add_lock.fetch_add(acquired_permits << 1, Ordering::AcqRel); + self.semaphore.add_permits_locked(acquired_permits, waiters, false); + } + } else { drop(waiters); }; @@ -562,10 +594,6 @@ impl AcquireError { } } -fn to_try_acquire(_: AcquireError) -> TryAcquireError { - TryAcquireError::Closed -} - impl fmt::Display for AcquireError { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { write!(fmt, "semaphore closed") @@ -610,6 +638,11 @@ impl std::error::Error for TryAcquireError {} /// /// `Waiter` is forced to be !Unpin. unsafe impl linked_list::Link for Waiter { + // XXX: ideally, we would be able to use `Pin` here, to enforce the invariant that list entries + // may not move while in the list. However, we can't do this currently, as using `Pin<&'a mut + // Waiter>` as the `Handle` type would require `Semaphore` to be generic over a lifetime. We + // can't use `Pin<*mut Waiter>`, as raw pointers are `Unpin` regardless of whether or not they + // dereference to an `!Unpin` target. type Handle = NonNull; type Target = Waiter; From 9f49f807eec418e8598147e817f770b0faa1fd41 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 13 Mar 2020 11:34:30 -0700 Subject: [PATCH 36/91] rm tests for stuff you can't do w new semaphore Signed-off-by: Eliza Weisman --- tokio/src/sync/tests/semaphore_batch.rs | 160 ------------------------ 1 file changed, 160 deletions(-) diff --git a/tokio/src/sync/tests/semaphore_batch.rs b/tokio/src/sync/tests/semaphore_batch.rs index 7fea0032cc8..8dc0993bff9 100644 --- a/tokio/src/sync/tests/semaphore_batch.rs +++ b/tokio/src/sync/tests/semaphore_batch.rs @@ -136,25 +136,6 @@ fn forget_acquired() { assert_eq!(s.available_permits(), 0); } -// #[test] -// fn forget_waiting() { -// let s = Semaphore::new(0); - -// // Polling for a permit succeeds immediately -// let mut permit = task::spawn(Box::new(Permit::new())); - -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - -// assert_eq!(s.available_permits(), 0); - -// permit.forget(1); - -// s.add_permits(1); - -// assert!(!permit.is_woken()); -// assert_eq!(s.available_permits(), 1); -// } - #[test] fn poll_acquire_many_unavailable() { let s = Semaphore::new(5); @@ -338,147 +319,6 @@ fn close_semaphore_notifies_permit2() { assert_eq!(2, s.available_permits()); } -// #[test] -// fn poll_acquire_additional_permits_while_waiting_before_assigned() { -// let s = Semaphore::new(1); - -// let mut permit = task::spawn(Box::new(Permit::new())); - -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); - -// s.add_permits(1); -// assert!(!permit.is_woken()); - -// s.add_permits(1); -// assert!(permit.is_woken()); - -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); -// } - -// #[test] -// fn try_acquire_additional_permits_while_waiting_before_assigned() { -// let s = Semaphore::new(1); - -// let mut permit = task::spawn(Box::new(Permit::new())); - -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - -// assert_err!(permit.enter(|_, mut p| p.try_acquire(3, &s))); - -// s.add_permits(1); -// assert!(permit.is_woken()); - -// assert_ok!(permit.enter(|_, mut p| p.try_acquire(2, &s))); -// } - -// #[test] -// fn poll_acquire_additional_permits_while_waiting_after_assigned_success() { -// let s = Semaphore::new(1); - -// let mut permit = task::spawn(Box::new(Permit::new())); - -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - -// s.add_permits(2); - -// assert!(permit.is_woken()); -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); -// } - -// #[test] -// fn poll_acquire_additional_permits_while_waiting_after_assigned_requeue() { -// let s = Semaphore::new(1); - -// let mut permit = task::spawn(Box::new(Permit::new())); - -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); - -// s.add_permits(2); - -// assert!(permit.is_woken()); -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); - -// s.add_permits(1); - -// assert!(permit.is_woken()); -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 4, &s))); -// } - -// #[test] -// fn poll_acquire_fewer_permits_while_waiting() { -// let s = Semaphore::new(1); - -// let mut permit = task::spawn(Box::new(Permit::new())); - -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// assert_eq!(s.available_permits(), 0); - -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -// assert_eq!(s.available_permits(), 0); -// } - -// #[test] -// fn poll_acquire_fewer_permits_after_assigned() { -// let s = Semaphore::new(1); - -// let mut permit1 = task::spawn(Box::new(Permit::new())); -// let mut permit2 = task::spawn(Box::new(Permit::new())); - -// assert_pending!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 5, &s))); -// assert_eq!(s.available_permits(), 0); - -// assert_pending!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - -// s.add_permits(4); -// assert!(permit1.is_woken()); -// assert!(!permit2.is_woken()); - -// assert_ready_ok!(permit1.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 3, &s))); - -// assert!(permit2.is_woken()); -// assert_eq!(s.available_permits(), 1); - -// assert_ready_ok!(permit2.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); -// } - -// #[test] -// fn forget_partial_1() { -// let s = Semaphore::new(0); - -// let mut permit = task::spawn(Box::new(Permit::new())); - -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// s.add_permits(1); - -// assert_eq!(0, s.available_permits()); - -// permit.release(1, &s); - -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 1, &s))); - -// assert_eq!(s.available_permits(), 0); -// } - -// #[test] -// fn forget_partial_2() { -// let s = Semaphore::new(0); - -// let mut permit = task::spawn(Box::new(Permit::new())); - -// assert_pending!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// s.add_permits(1); - -// assert_eq!(0, s.available_permits()); - -// permit.release(1, &s); - -// s.add_permits(1); - -// assert_ready_ok!(permit.enter(|cx, mut p| p.as_mut().poll_acquire(cx, 2, &s))); -// assert_eq!(s.available_permits(), 0); -// } - #[test] fn cancel_acquire_releases_permits() { let s = Semaphore::new(10); From fea9fd250280c7d42d1e74e61bd293594f32c301 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 13 Mar 2020 15:32:53 -0700 Subject: [PATCH 37/91] clean up & add comments Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 111 ++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 35 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 5e3af3a5431..b734863658b 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -39,13 +39,17 @@ pub(crate) struct Semaphore { waiters: Mutex, /// The current number of available permits in the semaphore. permits: AtomicUsize, + /// Permits in the process of being released. /// /// When releasing permits, if the lock on the semaphore's wait list is held by another task, /// the task releasing permits adds them to this counter. The task holding the lock will /// continue releasing permits until the counter is drained, allowing the `release` operation /// to be wait free. - add_lock: AtomicUsize, + /// + /// The first bit of this counter indicates that the semaphore is closing. Therefore, all + /// values are shifted over one bit. + adding: AtomicUsize, } struct Waitlist { @@ -86,11 +90,27 @@ enum PermitState { Acquired(u16), } +/// An entry in the wait queue. struct Waiter { - waker: CausalCell>, + /// The current state of the waiter. + /// + /// This is either the number of remaining permits required by + /// the waiter, or a flag indicating that the waiter is not yet queued. state: AtomicUsize, - /// Intrusive linked-list pointers + /// The waker to notify the task awaiting permits. + /// + /// # Safety + /// + /// This may only be accessed while the wait queue is locked. + /// XXX: it would be nice if we could enforce this... + waker: CausalCell>, + + /// Intrusive linked-list pointers. + /// + /// # Safety + /// + /// This may only be accessed while the wait queue is locked. pointers: linked_list::Pointers, /// Should not be `Unpin`. @@ -109,7 +129,7 @@ impl Semaphore { queue: LinkedList::new(), closed: false, }), - add_lock: AtomicUsize::new(0), + adding: AtomicUsize::new(0), } } @@ -118,54 +138,73 @@ impl Semaphore { self.permits.load(Ordering::SeqCst) & std::u16::MAX as usize } - /// Closes the semaphore. This prevents the semaphore from issuing new - /// permits and notifies all pending waiters. - pub(crate) fn close(&self) { - self.permits.fetch_or(CLOSED, Ordering::Release); - // Acquire the `add_lock`, setting the "closed" flag on the lock. - let prev = self.add_lock.fetch_or(1, Ordering::AcqRel); - - if prev != 0 { - // Another thread has the lock and will be responsible for notifying - // pending waiters. - return; - } - - let mut lock = self.waiters.lock().unwrap(); - lock.closed = true; - self.add_permits_locked(0, lock, true); - } /// Adds `n` new permits to the semaphore. pub(crate) fn add_permits(&self, added: usize) { + // Assigning permits to waiters requires locking the wait list, so that any waiters which + // receive their required number of permits can be dequeued and notified. However, if + // multiple threads attempt to add permits concurrently, we are able to make this operation + // wait-free. By tracking an atomic counter of permits added to the semaphore, we are able + // to determine if another thread is currently holding the lock to assign permits. If + // this is the case, we simply add our permits to the counter and return, so we don't need + // to acquire the lock. Otherwise, no other thread is adding permits, so we lock the wait + // and loop until the counter of added permits is drained. + if added == 0 { return; } // TODO: Handle overflow. A panic is not sufficient, the process must // abort. - let prev = self.add_lock.fetch_add(added << 1, Ordering::AcqRel); + let prev = self.adding.fetch_add(added << 1, Ordering::AcqRel); if prev > 0 { // Another thread is already assigning permits. It will continue to do so until - // `add_lock` is drained, so we don't need to block on the lock. + // `added` is drained, so we don't need to block on the lock. return; } // Otherwise, no other thread is assigning permits. Thus, we must lock the semaphore's wait - // list and assign permits until `add_lock` is drained. + // list and assign permits until `added` is drained. self.add_permits_locked(added, self.waiters.lock().unwrap(), false); } + /// Closes the semaphore. This prevents the semaphore from issuing new + /// permits and notifies all pending waiters. + pub(crate) fn close(&self) { + // Closing the semaphore works similarly to adding permits: if another thread is already + // holding the lock on the wait list to assign permits, we simply set a bit in the counter + // of permits being added. The thread holding the lock will see that bit has been set, and + // begin closing the semaphore. + + self.permits.fetch_or(CLOSED, Ordering::Release); + // Acquire the `added`, setting the "closed" flag on the lock. + let prev = self.adding.fetch_or(1, Ordering::AcqRel); + + if prev != 0 { + // Another thread has the lock and will be responsible for notifying + // pending waiters. + return; + } + + let mut lock = self.waiters.lock().unwrap(); + lock.closed = true; + self.add_permits_locked(0, lock, true); + } + fn add_permits_locked( &self, mut rem: usize, mut waiters: MutexGuard<'_, Waitlist>, mut closed: bool, ) { + // The thread adding permits will loop until the counter of added permits is drained. If + // threads add permits (or close the semaphore) while we are holding the lock, we will add + // those permits as well, so that the other thread does not need to block. + loop { - // how many permits are we releasing on this pass? + // How many permits are we releasing on this iteration? let initial = rem; - // Release the permits and notify + // Assign permits to the wait queue and notify any satisfied waiters. while rem > 0 || waiters.closed { let pop = match waiters.queue.last() { Some(_) if waiters.closed => true, @@ -177,10 +216,11 @@ impl Semaphore { }; if pop { let node = waiters.queue.pop_back().unwrap(); - let waiter = unsafe { node.as_ref() }; - let waker = waiter.waker.with_mut(|waker| unsafe { - (*waker).take() - }); + // Safety: we are holding the lock on the wait queue, and thus have exclusive + // access to the nodes' wakers. + let waker = unsafe { + node.as_ref().waker.with_mut(|waker| (*waker).take()) + }; waker.expect("if a node is in the wait list, it must have a waker").wake(); @@ -190,12 +230,12 @@ impl Semaphore { let n = initial << 1; let actual = if closed { - let actual = self.add_lock.fetch_sub(n | 1, Ordering::AcqRel); + let actual = self.adding.fetch_sub(n | 1, Ordering::AcqRel); assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); closed = false; actual >> 1 } else { - let actual = self.add_lock.fetch_sub(n, Ordering::AcqRel); + let actual = self.adding.fetch_sub(n, Ordering::AcqRel); assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); if actual & 1 == 1 { waiters.closed = true; @@ -356,8 +396,8 @@ impl Drop for Semaphore { impl fmt::Debug for Semaphore { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_struct("Semaphore") - .field("permits", &self.add_lock.load(Ordering::Relaxed)) - .field("add_lock", &self.add_lock.load(Ordering::Relaxed)) + .field("permits", &self.adding.load(Ordering::Relaxed)) + .field("added", &self.adding.load(Ordering::Relaxed)) .finish() } } @@ -463,6 +503,7 @@ impl Default for Permit { impl Waiter { const UNQUEUED: usize = 1 << 16; + fn new() -> Self { Waiter { waker: CausalCell::new(None), @@ -571,7 +612,7 @@ impl Drop for Acquire<'_> { // we have already locked the mutex, so we know we will be the one to release these // permits, but it's necessary to add them since we will try to subtract them once we have // finished permit assignment. - self.semaphore.add_lock.fetch_add(acquired_permits << 1, Ordering::AcqRel); + self.semaphore.added.fetch_add(acquired_permits << 1, Ordering::AcqRel); self.semaphore.add_permits_locked(acquired_permits, waiters, false); } From df6c55692d1631e9a764f3230c0b3d4ae9f7ab29 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 09:01:08 -0700 Subject: [PATCH 38/91] rewrap comments, fix unused warnings Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 101 +++++++++++++++++------------- tokio/src/sync/semaphore_ll.rs | 1 + 2 files changed, 60 insertions(+), 42 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index b734863658b..c6db155d0d9 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -141,14 +141,16 @@ impl Semaphore { /// Adds `n` new permits to the semaphore. pub(crate) fn add_permits(&self, added: usize) { - // Assigning permits to waiters requires locking the wait list, so that any waiters which - // receive their required number of permits can be dequeued and notified. However, if - // multiple threads attempt to add permits concurrently, we are able to make this operation - // wait-free. By tracking an atomic counter of permits added to the semaphore, we are able - // to determine if another thread is currently holding the lock to assign permits. If - // this is the case, we simply add our permits to the counter and return, so we don't need - // to acquire the lock. Otherwise, no other thread is adding permits, so we lock the wait - // and loop until the counter of added permits is drained. + // Assigning permits to waiters requires locking the wait list, so that + // any waiters which receive their required number of permits can be + // dequeued and notified. However, if multiple threads attempt to add + // permits concurrently, we are able to make this operation wait-free. + // By tracking an atomic counter of permits added to the semaphore, we + // are able to determine if another thread is currently holding the lock + // to assign permits. If this is the case, we simply add our permits to + // the counter and return, so we don't need to acquire the lock. + // Otherwise, no other thread is adding permits, so we lock the wait and + // loop until the counter of added permits is drained. if added == 0 { return; @@ -158,23 +160,26 @@ impl Semaphore { // abort. let prev = self.adding.fetch_add(added << 1, Ordering::AcqRel); if prev > 0 { - // Another thread is already assigning permits. It will continue to do so until - // `added` is drained, so we don't need to block on the lock. + // Another thread is already assigning permits. It will continue to + // do so until `added` is drained, so we don't need to block on the + // lock. return; } - // Otherwise, no other thread is assigning permits. Thus, we must lock the semaphore's wait - // list and assign permits until `added` is drained. + // Otherwise, no other thread is assigning permits. Thus, we must lock + // the semaphore's wait list and assign permits until `added` is + // drained. self.add_permits_locked(added, self.waiters.lock().unwrap(), false); } /// Closes the semaphore. This prevents the semaphore from issuing new /// permits and notifies all pending waiters. pub(crate) fn close(&self) { - // Closing the semaphore works similarly to adding permits: if another thread is already - // holding the lock on the wait list to assign permits, we simply set a bit in the counter - // of permits being added. The thread holding the lock will see that bit has been set, and - // begin closing the semaphore. + // Closing the semaphore works similarly to adding permits: if another + // thread is already holding the lock on the wait list to assign + // permits, we simply set a bit in the counter of permits being added. + // The thread holding the lock will see that bit has been set, and begin + // closing the semaphore. self.permits.fetch_or(CLOSED, Ordering::Release); // Acquire the `added`, setting the "closed" flag on the lock. @@ -197,14 +202,16 @@ impl Semaphore { mut waiters: MutexGuard<'_, Waitlist>, mut closed: bool, ) { - // The thread adding permits will loop until the counter of added permits is drained. If - // threads add permits (or close the semaphore) while we are holding the lock, we will add - // those permits as well, so that the other thread does not need to block. + // The thread adding permits will loop until the counter of added + // permits is drained. If threads add permits (or close the semaphore) + // while we are holding the lock, we will add those permits as well, so + // that the other thread does not need to block. loop { // How many permits are we releasing on this iteration? let initial = rem; - // Assign permits to the wait queue and notify any satisfied waiters. + // Assign permits to the wait queue and notify any satisfied + // waiters. while rem > 0 || waiters.closed { let pop = match waiters.queue.last() { Some(_) if waiters.closed => true, @@ -216,8 +223,8 @@ impl Semaphore { }; if pop { let node = waiters.queue.pop_back().unwrap(); - // Safety: we are holding the lock on the wait queue, and thus have exclusive - // access to the nodes' wakers. + // Safety: we are holding the lock on the wait queue, and + // thus have exclusive access to the nodes' wakers. let waker = unsafe { node.as_ref().waker.with_mut(|waker| (*waker).take()) }; @@ -286,12 +293,14 @@ impl Semaphore { let mut state; let mut acquired = 0; - // First, try to take the requested number of permits from the semaphore. + // First, try to take the requested number of permits from the + // semaphore. let mut lock = None; let mut curr = self.permits.load(Ordering::Acquire); let mut waiters = loop { state = node.state.load(Ordering::Acquire); - // Has the waiter already acquired all its needed permits? If so, we're done! + // Has the waiter already acquired all its needed permits? If so, + // we're done! if state == 0 { return Ready(Ok(())); } @@ -312,12 +321,14 @@ impl Semaphore { }; if remaining > 0 && lock.is_none() { - // No permits were immediately available, so this permit will (probably) need to wait. - // We'll need to acquire a lock on the wait queue. + // No permits were immediately available, so this permit will + // (probably) need to wait. We'll need to acquire a lock on the + // wait queue. lock = Some(self.waiters.lock().unwrap()); - // While we were waiting to lock the wait list, additional permits may have been - // released. Therefore, we will acquire a new snapshot of the current state of the - // semaphore before actually enqueueing the waiter.. + // While we were waiting to lock the wait list, additional + // permits may have been released. Therefore, we will acquire a + // new snapshot of the current state of the semaphore before + // actually enqueueing the waiter.. continue; } @@ -344,7 +355,8 @@ impl Semaphore { } let next = match acquired.cmp(&state) { - // if the waiter is in the unqueued state, we need all the requested permits, minus the amount we just acquired. + // if the waiter is in the unqueued state, we need all the requested + // permits, minus the amount we just acquired. _ if state >= Waiter::UNQUEUED => needed as usize - acquired, // We have acquired all the needed permits! cmp::Ordering::Equal => 0, @@ -354,11 +366,12 @@ impl Semaphore { } }; - // Typically, one would probably expect to see a compare-and-swap loop here. However, in - // this case, we don't need to loop. Our most snapshot of the node's current state - // was acquired after we acquired the lock on the wait list. Because releasing permits tPPo - // waiters also requires locking the wait list, the state cannot have changed since we - // acquired this snapshot. + // Typically, one would probably expect to see a compare-and-swap loop + // here. However, in this case, we don't need to loop. Our most snapshot + // of the node's current state was acquired after we acquired the lock + // on the wait list. Because releasing permits tPPo waiters also + // requires locking the wait list, the state cannot have changed since + // we acquired this snapshot. node.state .compare_exchange(state, next, Ordering::AcqRel, Ordering::Acquire) .expect("state cannot have changed while the wait list was locked"); @@ -420,8 +433,9 @@ impl Permit { } } - /// Returns a future that tries to acquire the permit. If no permits are available, the current task - /// is notified once a new permit becomes available. + /// Returns a future that tries to acquire the permit. If no permits are + /// available, the current task is notified once a new permit becomes + /// available. pub(crate) fn acquire<'a>( &'a mut self, num_permits: u16, @@ -503,7 +517,7 @@ impl Default for Permit { impl Waiter { const UNQUEUED: usize = 1 << 16; - + fn new() -> Self { Waiter { waker: CausalCell::new(None), @@ -609,10 +623,11 @@ impl Drop for Acquire<'_> { unsafe { waiters.queue.remove(node) }; if acquired_permits > 0 { - // we have already locked the mutex, so we know we will be the one to release these - // permits, but it's necessary to add them since we will try to subtract them once we have - // finished permit assignment. - self.semaphore.added.fetch_add(acquired_permits << 1, Ordering::AcqRel); + // we have already locked the mutex, so we know we will be the + // one to release these permits, but it's necessary to add them + // since we will try to subtract them once we have finished + // permit assignment. + self.semaphore.adding.fetch_add(acquired_permits << 1, Ordering::AcqRel); self.semaphore.add_permits_locked(acquired_permits, waiters, false); } @@ -647,6 +662,7 @@ impl std::error::Error for AcquireError {} impl TryAcquireError { /// Returns `true` if the error was caused by a closed semaphore. + #[allow(dead_code)] // may be used later! pub(crate) fn is_closed(&self) -> bool { match self { TryAcquireError::Closed => true, @@ -656,6 +672,7 @@ impl TryAcquireError { /// Returns `true` if the error was caused by calling `try_acquire` on a /// semaphore with no available permits. + #[allow(dead_code)] // may be used later! pub(crate) fn is_no_permits(&self) -> bool { match self { TryAcquireError::NoPermits => true, diff --git a/tokio/src/sync/semaphore_ll.rs b/tokio/src/sync/semaphore_ll.rs index 69fd4a6a5d3..b56f21a8135 100644 --- a/tokio/src/sync/semaphore_ll.rs +++ b/tokio/src/sync/semaphore_ll.rs @@ -610,6 +610,7 @@ impl Permit { } /// Returns `true` if the permit has been acquired + #[allow(dead_code)] // may be used later pub(crate) fn is_acquired(&self) -> bool { match self.state { PermitState::Acquired(num) if num > 0 => true, From 201db12ad177b1573e63eff57a274cada9981caa Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 09:04:11 -0700 Subject: [PATCH 39/91] more wrapping Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 64 +++++++++++++++++-------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index c6db155d0d9..a04d49c9227 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -1,19 +1,23 @@ //! # Implementation Details -//! -//! The semaphore is implemented using an intrusive linked list of waiters. An atomic counter -//! tracks the number of available permits. If the semaphore does not contain the required number -//! of permits, the task attempting to acquire permits places its waker at the end of a queue. When -//! new permits are made available (such as by releasing an initial acquisition), they are assigned -//! to the task at the front of the queue, waking that task if its requested number of permits is -//! met. -//! -//! Because waiters are enqueued at the back of the linked list and dequeued from the front, the -//! semaphore is fair. Tasks trying to acquire large numbers of permits at a time will always be -//! woken eventually, even if many other tasks are acquiring smaller numbers of permits. This means -//! that in a use-case like tokio's read-write lock, writers will not be starved by readers. -//! -//! The linked list is guarded by a mutex, which must be acquired before enqueueing or dequeueing a -//! task. However, some operations are always wait-free. +//! +//! The semaphore is implemented using an intrusive linked list of waiters. An +//! atomic counter tracks the number of available permits. If the semaphore does +//! not contain the required number of permits, the task attempting to acquire +//! permits places its waker at the end of a queue. When new permits are made +//! available (such as by releasing an initial acquisition), they are assigned +//! to the task at the front of the queue, waking that task if its requested +//! number of permits is met. +//! +//! Because waiters are enqueued at the back of the linked list and dequeued +//! from the front, the semaphore is fair. Tasks trying to acquire large numbers +//! of permits at a time will always be woken eventually, even if many other +//! tasks are acquiring smaller numbers of permits. This means that in a +//! use-case like tokio's read-write lock, writers will not be starved by +//! readers. +//! +//! The linked list is guarded by a mutex, which must be acquired before +//! enqueueing or dequeueing a task. However, some operations are always +//! wait-free. use crate::loom::{ cell::CausalCell, sync::{atomic::AtomicUsize, Mutex, MutexGuard}, @@ -41,14 +45,14 @@ pub(crate) struct Semaphore { permits: AtomicUsize, /// Permits in the process of being released. - /// - /// When releasing permits, if the lock on the semaphore's wait list is held by another task, - /// the task releasing permits adds them to this counter. The task holding the lock will - /// continue releasing permits until the counter is drained, allowing the `release` operation - /// to be wait free. - /// - /// The first bit of this counter indicates that the semaphore is closing. Therefore, all - /// values are shifted over one bit. + /// + /// When releasing permits, if the lock on the semaphore's wait list is held + /// by another task, the task releasing permits adds them to this counter. + /// The task holding the lock will continue releasing permits until the + /// counter is drained, allowing the `release` operation to be wait free. + /// + /// The first bit of this counter indicates that the semaphore is closing. + /// Therefore, all values are shifted over one bit. adding: AtomicUsize, } @@ -635,7 +639,8 @@ impl Drop for Acquire<'_> { drop(waiters); }; - // If the permit had transitioned to the `Waiting` state, put it back into `Acquired`. + // If the permit had transitioned to the `Waiting` state, put it back + // into `Acquired`. if let PermitState::Waiting(_) = self.permit.state { self.permit.state = PermitState::Acquired(0); } @@ -696,11 +701,12 @@ impl std::error::Error for TryAcquireError {} /// /// `Waiter` is forced to be !Unpin. unsafe impl linked_list::Link for Waiter { - // XXX: ideally, we would be able to use `Pin` here, to enforce the invariant that list entries - // may not move while in the list. However, we can't do this currently, as using `Pin<&'a mut - // Waiter>` as the `Handle` type would require `Semaphore` to be generic over a lifetime. We - // can't use `Pin<*mut Waiter>`, as raw pointers are `Unpin` regardless of whether or not they - // dereference to an `!Unpin` target. + // XXX: ideally, we would be able to use `Pin` here, to enforce the + // invariant that list entries may not move while in the list. However, we + // can't do this currently, as using `Pin<&'a mut Waiter>` as the `Handle` + // type would require `Semaphore` to be generic over a lifetime. We can't + // use `Pin<*mut Waiter>`, as raw pointers are `Unpin` regardless of whether + // or not they dereference to an `!Unpin` target. type Handle = NonNull; type Target = Waiter; From 9f3f544144a99ff78c03086ee71861ad39d97eb5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 09:08:28 -0700 Subject: [PATCH 40/91] style/clean up Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 53 ++++++++++++++++--------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index a04d49c9227..01be9c4fbb1 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -30,7 +30,7 @@ use std::{ marker::PhantomPinned, pin::Pin, ptr::NonNull, - sync::atomic::Ordering, + sync::atomic::Ordering::*, task::{ Context, Poll, Poll::{Pending, Ready}, @@ -139,7 +139,7 @@ impl Semaphore { /// Returns the current number of available permits pub(crate) fn available_permits(&self) -> usize { - self.permits.load(Ordering::SeqCst) & std::u16::MAX as usize + self.permits.load(SeqCst) & std::u16::MAX as usize } @@ -162,7 +162,7 @@ impl Semaphore { // TODO: Handle overflow. A panic is not sufficient, the process must // abort. - let prev = self.adding.fetch_add(added << 1, Ordering::AcqRel); + let prev = self.adding.fetch_add(added << 1, AcqRel); if prev > 0 { // Another thread is already assigning permits. It will continue to // do so until `added` is drained, so we don't need to block on the @@ -185,9 +185,9 @@ impl Semaphore { // The thread holding the lock will see that bit has been set, and begin // closing the semaphore. - self.permits.fetch_or(CLOSED, Ordering::Release); + self.permits.fetch_or(CLOSED, Release); // Acquire the `added`, setting the "closed" flag on the lock. - let prev = self.adding.fetch_or(1, Ordering::AcqRel); + let prev = self.adding.fetch_or(1, AcqRel); if prev != 0 { // Another thread has the lock and will be responsible for notifying @@ -221,7 +221,7 @@ impl Semaphore { Some(_) if waiters.closed => true, Some(last) => last.assign_permits(&mut rem, waiters.closed), None => { - self.permits.fetch_add(rem, Ordering::AcqRel); + self.permits.fetch_add(rem, AcqRel); break; } }; @@ -241,12 +241,12 @@ impl Semaphore { let n = initial << 1; let actual = if closed { - let actual = self.adding.fetch_sub(n | 1, Ordering::AcqRel); + let actual = self.adding.fetch_sub(n | 1, AcqRel); assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); closed = false; actual >> 1 } else { - let actual = self.adding.fetch_sub(n, Ordering::AcqRel); + let actual = self.adding.fetch_sub(n, AcqRel); assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); if actual & 1 == 1 { waiters.closed = true; @@ -264,7 +264,7 @@ impl Semaphore { } fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { - let mut curr = self.permits.load(Ordering::Acquire); + let mut curr = self.permits.load(Acquire); loop { // Has the semaphore closed? if curr & CLOSED > 0 { @@ -280,7 +280,7 @@ impl Semaphore { match self .permits - .compare_exchange(curr, next, Ordering::AcqRel, Ordering::Acquire) + .compare_exchange(curr, next, AcqRel, Acquire) { Ok(_) => return Ok(()), Err(actual) => curr = actual, @@ -300,9 +300,9 @@ impl Semaphore { // First, try to take the requested number of permits from the // semaphore. let mut lock = None; - let mut curr = self.permits.load(Ordering::Acquire); + let mut curr = self.permits.load(Acquire); let mut waiters = loop { - state = node.state.load(Ordering::Acquire); + state = node.state.load(Acquire); // Has the waiter already acquired all its needed permits? If so, // we're done! if state == 0 { @@ -339,8 +339,8 @@ impl Semaphore { match self.permits.compare_exchange_weak( curr, next, - Ordering::AcqRel, - Ordering::Acquire, + AcqRel, + Acquire, ) { Ok(_) => { acquired += acq; @@ -377,7 +377,7 @@ impl Semaphore { // requires locking the wait list, the state cannot have changed since // we acquired this snapshot. node.state - .compare_exchange(state, next, Ordering::AcqRel, Ordering::Acquire) + .compare_exchange(state, next, AcqRel, Acquire) .expect("state cannot have changed while the wait list was locked"); if next == 0 { @@ -413,8 +413,8 @@ impl Drop for Semaphore { impl fmt::Debug for Semaphore { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_struct("Semaphore") - .field("permits", &self.adding.load(Ordering::Relaxed)) - .field("added", &self.adding.load(Ordering::Relaxed)) + .field("permits", &self.adding.load(Relaxed)) + .field("added", &self.adding.load(Relaxed)) .finish() } } @@ -540,7 +540,7 @@ impl Waiter { return true; } - let mut curr = self.state.load(Ordering::Acquire); + let mut curr = self.state.load(Acquire); loop { // Assign up to `n` permits. @@ -548,7 +548,7 @@ impl Waiter { let next = curr - assign; match self .state - .compare_exchange(curr, next, Ordering::AcqRel, Ordering::Acquire) + .compare_exchange(curr, next, AcqRel, Acquire) { Ok(_) => { *n = n.saturating_sub(assign); @@ -573,9 +573,9 @@ impl Future for Acquire<'_> { ready!(semaphore.poll_acquire(cx, needed - n, node))?; PermitState::Acquired(needed) } - PermitState::Waiting(_n) => { - assert_eq!(_n, needed, "how the heck did you get in this state?"); - if node.state.load(Ordering::Acquire) > 0 { + PermitState::Waiting(n) => { + assert_eq!(n, needed, "permit cannot be modified while waiting"); + if node.state.load(Acquire) > 0 { ready!(semaphore.poll_acquire(cx, needed, node))?; } PermitState::Acquired(needed) @@ -608,7 +608,7 @@ impl Acquire<'_> { impl Drop for Acquire<'_> { fn drop(&mut self) { - let state = self.node.state.load(Ordering::Acquire); + let state = self.node.state.load(Acquire); // fast path: if we aren't actually waiting, no need to acquire the lock. if state & Waiter::UNQUEUED == Waiter::UNQUEUED { return; @@ -631,13 +631,16 @@ impl Drop for Acquire<'_> { // one to release these permits, but it's necessary to add them // since we will try to subtract them once we have finished // permit assignment. - self.semaphore.adding.fetch_add(acquired_permits << 1, Ordering::AcqRel); + self.semaphore.adding.fetch_add(acquired_permits << 1, AcqRel); self.semaphore.add_permits_locked(acquired_permits, waiters, false); } } else { + // We don't need to continue holding the lock while modifying the + // permit's local state. drop(waiters); - }; + } + // If the permit had transitioned to the `Waiting` state, put it back // into `Acquired`. From 8628723cbfe704f9ad66c5f86184f408c272de29 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 12:03:05 -0700 Subject: [PATCH 41/91] revert unneeded MPSC changes Signed-off-by: Eliza Weisman --- tokio/src/sync/mpsc/bounded.rs | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/tokio/src/sync/mpsc/bounded.rs b/tokio/src/sync/mpsc/bounded.rs index f289ab95531..ba89fdd72cc 100644 --- a/tokio/src/sync/mpsc/bounded.rs +++ b/tokio/src/sync/mpsc/bounded.rs @@ -10,14 +10,11 @@ cfg_time! { use std::fmt; use std::task::{Context, Poll}; -pin_project_lite::pin_project! { - /// Send values to the associated `Receiver`. - /// - /// Instances are created by the [`channel`](channel) function. - pub struct Sender { - #[pin] - chan: chan::Tx, - } +/// Send values to the associated `Receiver`. +/// +/// Instances are created by the [`channel`](channel) function. +pub struct Sender { + chan: chan::Tx, } impl Clone for Sender { @@ -201,9 +198,7 @@ impl Sender { #[doc(hidden)] // TODO: document pub fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - unsafe { std::pin::Pin::new_unchecked(&mut self.chan) } - .poll_ready(cx) - .map_err(|_| ClosedError::new()) + self.chan.poll_ready(cx).map_err(|_| ClosedError::new()) } /// Attempts to immediately send a message on this `Sender` @@ -320,10 +315,8 @@ impl Sender { /// ``` pub async fn send(&mut self, value: T) -> Result<(), SendError> { use crate::future::poll_fn; - if poll_fn(|cx| unsafe { std::pin::Pin::new_unchecked(&mut self.chan) }.poll_ready(cx)) - .await - .is_err() - { + + if poll_fn(|cx| self.poll_ready(cx)).await.is_err() { return Err(SendError(value)); } From 49ed5dd521809735be665ae81d8319c89f978fb5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 12:07:21 -0700 Subject: [PATCH 42/91] revert unneeded MPSC test changes Signed-off-by: Eliza Weisman --- tokio/tests/sync_mpsc.rs | 65 ++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/tokio/tests/sync_mpsc.rs b/tokio/tests/sync_mpsc.rs index ab8ab386c6d..7e5c60e2e2f 100644 --- a/tokio/tests/sync_mpsc.rs +++ b/tokio/tests/sync_mpsc.rs @@ -15,31 +15,30 @@ trait AssertSend: Send {} impl AssertSend for mpsc::Sender {} impl AssertSend for mpsc::Receiver {} -// #[test] -// fn send_recv_with_buffer() { -// let (tx, rx) = mpsc::channel::(16); -// let mut tx = task::spawn(tx); -// let mut rx = task::spawn(rx); +#[test] +fn send_recv_with_buffer() { + let (tx, rx) = mpsc::channel::(16); + let mut tx = task::spawn(tx); + let mut rx = task::spawn(rx); -// // Using poll_ready / try_send -// assert_ready_ok!(tx.enter(|cx, mut tx| tx.poll_ready(cx))); -// // tx.try_send(1).unwrap(); -// unimplemented!(); + // Using poll_ready / try_send + assert_ready_ok!(tx.enter(|cx, mut tx| tx.poll_ready(cx))); + tx.try_send(1).unwrap(); -// // Without poll_ready -// // tx.try_send(2).unwrap(); + // Without poll_ready + tx.try_send(2).unwrap(); -// drop(tx); + drop(tx); -// let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); -// assert_eq!(val, Some(1)); + let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); + assert_eq!(val, Some(1)); -// let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); -// assert_eq!(val, Some(2)); + let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); + assert_eq!(val, Some(2)); -// let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); -// assert!(val.is_none()); -// } + let val = assert_ready!(rx.enter(|cx, mut rx| rx.poll_recv(cx))); + assert!(val.is_none()); +} #[tokio::test] async fn send_recv_stream_with_buffer() { @@ -162,23 +161,23 @@ async fn send_recv_stream_unbounded() { assert_eq!(None, rx.next().await); } -// #[test] -// fn no_t_bounds_buffer() { -// struct NoImpls; +#[test] +fn no_t_bounds_buffer() { + struct NoImpls; -// let mut t1 = task::spawn(()); -// let (tx, mut rx) = mpsc::channel(100); + let mut t1 = task::spawn(()); + let (tx, mut rx) = mpsc::channel(100); -// // sender should be Debug even though T isn't Debug -// println!("{:?}", tx); -// // same with Receiver -// println!("{:?}", rx); -// // and sender should be Clone even though T isn't Clone -// assert!(tx.clone().try_send(NoImpls).is_ok()); + // sender should be Debug even though T isn't Debug + println!("{:?}", tx); + // same with Receiver + println!("{:?}", rx); + // and sender should be Clone even though T isn't Clone + assert!(tx.clone().try_send(NoImpls).is_ok()); -// let val = assert_ready!(t1.enter(|cx, _| rx.poll_recv(cx))); -// assert!(val.is_some()); -// } + let val = assert_ready!(t1.enter(|cx, _| rx.poll_recv(cx))); + assert!(val.is_some()); +} #[test] fn no_t_bounds_unbounded() { From f5f5bc11e87be7c43e08ea163c96f9e78ab78f30 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 12:22:12 -0700 Subject: [PATCH 43/91] Apply suggestions from code review Co-Authored-By: Carl Lerche --- tokio/src/sync/tests/loom_semaphore_batch.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index fe13ae16857..135cdd98d15 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -35,7 +35,6 @@ fn basic_usage() { loom::model(|| { - println!("\n ------ iter ------ \n"); let shared = Arc::new(Shared { semaphore: Semaphore::new(NUM), active: AtomicUsize::new(0), @@ -190,4 +189,4 @@ fn release_during_acquire() { permit2.release(4, &semaphore); assert_eq!(10, semaphore.available_permits()); }) -} \ No newline at end of file +} From 24479c5454a417314e2a84de8b5da604891a840b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 13:11:54 -0700 Subject: [PATCH 44/91] remove printlns from tests thanks for noticing @carllerche Co-Authored-By: Carl Lerche --- tokio/src/sync/tests/loom_semaphore_batch.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index 135cdd98d15..4da45f344fb 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -23,7 +23,6 @@ fn basic_usage() { async fn actor(shared: Arc) { let mut permit = Permit::new(); permit.acquire(1, &shared.semaphore).await.unwrap(); - println!("acquired!"); let actual = shared.active.fetch_add(1, SeqCst); assert!(actual <= NUM - 1); @@ -81,7 +80,6 @@ fn basic_closing() { const NUM: usize = 2; loom::model(|| { - println!("-- iter --"); let semaphore = Arc::new(Semaphore::new(1)); for _ in 0..NUM { From 6275513f7a6526aeed61c1cec7f38497bddd9377 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 13:44:27 -0700 Subject: [PATCH 45/91] checks to ensure API types remain Send/Sync/Unpin Signed-off-by: Eliza Weisman --- tokio/src/sync/mutex.rs | 6 ++++-- tokio/src/sync/rwlock.rs | 17 +++++++++++++++++ tokio/src/sync/semaphore.rs | 8 ++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tokio/src/sync/mutex.rs b/tokio/src/sync/mutex.rs index 1af9eb0e6b1..a0487a13197 100644 --- a/tokio/src/sync/mutex.rs +++ b/tokio/src/sync/mutex.rs @@ -135,8 +135,10 @@ impl Error for TryLockError {} #[test] #[cfg(not(loom))] fn bounds() { - fn check() {} - check::>(); + fn check_send() {} + fn check_unpin() {} + check_send::>(); + check_unpin::>(); } impl Mutex { diff --git a/tokio/src/sync/rwlock.rs b/tokio/src/sync/rwlock.rs index d9d049ea953..2a08f9b84c3 100644 --- a/tokio/src/sync/rwlock.rs +++ b/tokio/src/sync/rwlock.rs @@ -123,6 +123,23 @@ impl<'a, T> Drop for ReleasingPermit<'a, T> { } } +#[test] +#[cfg(not(loom))] +fn bounds() { + fn check_send() {} + fn check_sync() {} + fn check_unpin() {} + check_send::>(); + check_sync::>(); + check_unpin::>(); + + check_sync::>(); + check_unpin::>(); + + check_sync::>(); + check_unpin::>(); +} + // As long as T: Send + Sync, it's fine to send and share RwLock between threads. // If T were not Send, sending and sharing a RwLock would be bad, since you can access T through // RwLock. diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index 5566fa79942..62bf2e7bf39 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -35,6 +35,14 @@ pub struct SemaphorePermit<'a> { #[derive(Debug)] pub struct TryAcquireError(()); +#[test] +#[cfg(not(loom))] +fn bounds() { + fn check_unpin() {} + check_unpin::(); + check_unpin::>(); +} + impl Semaphore { /// Creates a new semaphore with the initial number of permits pub fn new(permits: usize) -> Self { From f86147a37808c44df94b197836353f7a99422256 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 13:44:50 -0700 Subject: [PATCH 46/91] style: more consistent imports Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 01be9c4fbb1..34c95d22a6d 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -18,25 +18,18 @@ //! The linked list is guarded by a mutex, which must be acquired before //! enqueueing or dequeueing a task. However, some operations are always //! wait-free. -use crate::loom::{ - cell::CausalCell, - sync::{atomic::AtomicUsize, Mutex, MutexGuard}, -}; +use crate::loom::cell::CausalCell; +use crate::loom::sync::{atomic::AtomicUsize, Mutex, MutexGuard}; use crate::util::linked_list::{self, LinkedList}; -use std::{ - cmp, fmt, - future::Future, - marker::PhantomPinned, - pin::Pin, - ptr::NonNull, - sync::atomic::Ordering::*, - task::{ - Context, Poll, - Poll::{Pending, Ready}, - Waker, - }, -}; +use std::{cmp, fmt}; +use std::future::Future; +use std::marker::PhantomPinned; +use std::pin::Pin; +use std::ptr::NonNull; +use std::sync::atomic::Ordering::*; +use std::task::{Context, Poll, Waker}; +use std::task::Poll::*; /// An asynchronous counting semaphore which permits waiting on multiple permits at once. pub(crate) struct Semaphore { From 9206558eb5f0ea50b8ed0c43078be16526a2816f Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 13:45:12 -0700 Subject: [PATCH 47/91] make docs more accurate Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 34c95d22a6d..52da9cd9253 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -16,7 +16,7 @@ //! readers. //! //! The linked list is guarded by a mutex, which must be acquired before -//! enqueueing or dequeueing a task. However, some operations are always +//! enqueueing or dequeueing a task. However, some operations are often //! wait-free. use crate::loom::cell::CausalCell; use crate::loom::sync::{atomic::AtomicUsize, Mutex, MutexGuard}; From b41228450049d616d2d6e157f92b23b5f92a4b55 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 13:47:19 -0700 Subject: [PATCH 48/91] comments style improvements Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 52da9cd9253..746637eed25 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -352,7 +352,7 @@ impl Semaphore { } let next = match acquired.cmp(&state) { - // if the waiter is in the unqueued state, we need all the requested + // If the waiter is in the unqueued state, we need all the requested // permits, minus the amount we just acquired. _ if state >= Waiter::UNQUEUED => needed as usize - acquired, // We have acquired all the needed permits! @@ -377,14 +377,13 @@ impl Semaphore { return Ready(Ok(())); } - // otherwise, register the waker & enqueue the node. + // Otherwise, register the waker & enqueue the node. node.waker.with_mut(|waker| - // safety: the wait list is locked, so we may modify the waker. + // Safety: the wait list is locked, so we may modify the waker. unsafe { *waker = Some(cx.waker().clone()) } ); unsafe { - // XXX(eliza) T_T let node = Pin::into_inner_unchecked(node) as *mut _; let node = NonNull::new_unchecked(node); @@ -616,7 +615,7 @@ impl Drop for Acquire<'_> { let acquired_permits = self.num_permits as usize - state; // remove the entry from the list // - // safety: we have locked the wait list. + // Safety: we have locked the wait list. unsafe { waiters.queue.remove(node) }; if acquired_permits > 0 { From 3b4f41638332c61f7adfc7f3180a9eb2d45fa4aa Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 14:06:31 -0700 Subject: [PATCH 49/91] make LinkedList::is_linked more misuse-resistant Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 31 +++++++++++++++++++++++--- tokio/src/util/linked_list.rs | 37 ++++++++++++++++++------------- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 746637eed25..8586cf0aa44 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -366,7 +366,7 @@ impl Semaphore { // Typically, one would probably expect to see a compare-and-swap loop // here. However, in this case, we don't need to loop. Our most snapshot // of the node's current state was acquired after we acquired the lock - // on the wait list. Because releasing permits tPPo waiters also + // on the wait list. Because releasing permits to waiters also // requires locking the wait list, the state cannot have changed since // we acquired this snapshot. node.state @@ -387,7 +387,8 @@ impl Semaphore { let node = Pin::into_inner_unchecked(node) as *mut _; let node = NonNull::new_unchecked(node); - if !waiters.queue.is_linked(&node) { + // If the waiter is not already in the wait queue, enqueue it. + if !waiters.contains(node) { waiters.queue.push_front(node); } } @@ -610,7 +611,7 @@ impl Drop for Acquire<'_> { // in the linked list. let mut waiters = self.semaphore.waiters.lock().unwrap(); let node = NonNull::from(&mut self.node); - if waiters.queue.is_linked(&node) { + if waiters.contains(node) { let acquired_permits = self.num_permits as usize - state; // remove the entry from the list @@ -717,3 +718,27 @@ unsafe impl linked_list::Link for Waiter { NonNull::from(&mut target.as_mut().pointers) } } + +impl Waitlist { + /// Returns true if this waitlist already contains the given waiter + fn contains(&self, node: NonNull) -> bool { + use linked_list::Link; + // Note: `is_linked` does not necessarily indicate that the node is + // linked with _this_ list. However, because nodes are only + // added/removed inside of `Acquire` futures, and a reference to the + // same `Semaphore` is present whenever the `Acquire` future calls this + // we know that the node cannot be linked with another list. + if unsafe { Waiter::pointers(node).as_ref() }.is_linked() { + true + } else if self.queue.is_first(&node) { + debug_assert!( + self.queue.is_last(&node), + "if a node is unlinked but is the head of a queue, it must \ + also be the tail of the queue" + ); + true + } else { + false + } + } +} diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 5faf3d37df9..751558d4e7f 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -132,22 +132,14 @@ impl LinkedList { true } - pub(crate) fn is_linked(&self, node: &T::Handle) -> bool { - let node = T::as_raw(node); - unsafe { - if T::pointers(node).as_ref().next.is_some() - || T::pointers(node).as_ref().prev.is_some() - { - return true; - } - } - if let Some(head) = self.head { - if head == node { - assert_eq!(self.tail, Some(node)); - return true; - } - } - false + /// Returns `true` if `node` is the first node in this list. + pub(crate) fn is_first(&self, node: &T::Handle) -> bool { + self.head == Some(T::as_raw(node)) + } + + /// Returns `true` if `node` is the last node in this list. + pub(crate) fn is_last(&self, node: &T::Handle) -> bool { + self.tail == Some(T::as_raw(node)) } /// Removes the specified node from the list @@ -230,6 +222,19 @@ impl Pointers { next: None, } } + + /// Returns `true` if this set of `Pointers` is linked to a previous or next + /// node. + /// + /// # Notes + /// - This does _not_ test for membership in a given list; simply + /// whether the pointers are null or not. + /// - If a node is the _only_ node in a list, calling `is_linked` on its + /// `Pointers` will return `false`, but `LinkedList::is_first` and + /// `LinkedList::is_last` will return `true`. + pub(crate) fn is_linked(&self) -> bool { + self.prev.is_some() || self.next.is_some() + } } #[cfg(test)] From c2c3e048e8bb6f0b89e6fa4dbc552a46f06604bd Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 16 Mar 2020 14:13:24 -0700 Subject: [PATCH 50/91] fix feature flag unhappiness Signed-off-by: Eliza Weisman --- tokio/src/util/linked_list.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 751558d4e7f..fe6bfaeca9c 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -118,6 +118,7 @@ impl LinkedList { } } + #[cfg(feature = "sync")] // used only by `batch_semaphore` pub(crate) fn last<'a>(&'a self) -> Option<&'a T::Target> { unsafe { Some(&*self.tail?.as_ptr()) } } @@ -133,11 +134,13 @@ impl LinkedList { } /// Returns `true` if `node` is the first node in this list. + #[cfg(feature = "sync")] // used only by `batch_semaphore` pub(crate) fn is_first(&self, node: &T::Handle) -> bool { self.head == Some(T::as_raw(node)) } /// Returns `true` if `node` is the last node in this list. + #[cfg(feature = "sync")] // used only by `batch_semaphore` pub(crate) fn is_last(&self, node: &T::Handle) -> bool { self.tail == Some(T::as_raw(node)) } @@ -232,6 +235,7 @@ impl Pointers { /// - If a node is the _only_ node in a list, calling `is_linked` on its /// `Pointers` will return `false`, but `LinkedList::is_first` and /// `LinkedList::is_last` will return `true`. + #[cfg(feature = "sync")] // used only by `batch_semaphore` pub(crate) fn is_linked(&self) -> bool { self.prev.is_some() || self.next.is_some() } From c9e122f662630d5ce8d644896d260e86465f7e3b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 17 Mar 2020 10:28:32 -0700 Subject: [PATCH 51/91] quick rwlock benches Signed-off-by: Eliza Weisman --- benches/Cargo.toml | 6 ++ benches/sync_rwlock.rs | 152 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 benches/sync_rwlock.rs diff --git a/benches/Cargo.toml b/benches/Cargo.toml index f4a1d8fb71e..a02ee68f5a9 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -22,3 +22,9 @@ harness = false name = "scheduler" path = "scheduler.rs" harness = false + + +[[bench]] +name = "sync_rwlock" +path = "sync_rwlock.rs" +harness = false diff --git a/benches/sync_rwlock.rs b/benches/sync_rwlock.rs new file mode 100644 index 00000000000..c7b6395e1a3 --- /dev/null +++ b/benches/sync_rwlock.rs @@ -0,0 +1,152 @@ +use bencher::{black_box, Bencher}; +use tokio::{task, sync::RwLock}; +use std::sync::Arc; + +fn read_uncontended(b: &mut Bencher) { + let mut rt = tokio::runtime::Builder::new() + .core_threads(6) + .threaded_scheduler() + .build() + .unwrap(); + + let lock = Arc::new(RwLock::new(())); + b.iter(|| { + + let lock = lock.clone(); + rt.block_on(async move { + for _ in 0..6 { + let read = lock.read().await; + black_box(read); + } + }) + }); +} + +fn read_concurrent_uncontended_multi(b: &mut Bencher) { + let mut rt = tokio::runtime::Builder::new() + .core_threads(6) + .threaded_scheduler() + .build() + .unwrap(); + + async fn task(lock: Arc>) { + let read = lock.read().await; + black_box(read); + } + + let lock = Arc::new(RwLock::new(())); + b.iter(|| { + let lock = lock.clone(); + rt.block_on(async move { + let j = tokio::try_join!{ + task::spawn(task(lock.clone())), + task::spawn(task(lock.clone())), + task::spawn(task(lock.clone())), + task::spawn(task(lock.clone())), + task::spawn(task(lock.clone())), + task::spawn(task(lock.clone())) + }; + j.unwrap(); + + }) + }); +} + +fn read_concurrent_uncontended(b: &mut Bencher) { + let mut rt = tokio::runtime::Builder::new() + .basic_scheduler() + .build() + .unwrap(); + + async fn task(lock: Arc>) { + let read = lock.read().await; + black_box(read); + } + + let lock = Arc::new(RwLock::new(())); + b.iter(|| { + let lock = lock.clone(); + rt.block_on(async move { + tokio::join!{ + task(lock.clone()), + task(lock.clone()), + task(lock.clone()), + task(lock.clone()), + task(lock.clone()), + task(lock.clone()) + }; + + }) + }); +} + +fn read_concurrent_contended_multi(b: &mut Bencher) { + let mut rt = tokio::runtime::Builder::new() + .core_threads(6) + .threaded_scheduler() + .build() + .unwrap(); + + async fn task(lock: Arc>) { + let read = lock.read().await; + black_box(read); + } + + let lock = Arc::new(RwLock::new(())); + b.iter(|| { + let lock = lock.clone(); + rt.block_on(async move { + let write = lock.write().await; + let j = tokio::try_join!{ + async move { drop(write); Ok(()) }, + task::spawn(task(lock.clone())), + task::spawn(task(lock.clone())), + task::spawn(task(lock.clone())), + task::spawn(task(lock.clone())), + task::spawn(task(lock.clone())), + }; + j.unwrap(); + + }) + }); +} + +fn read_concurrent_contended(b: &mut Bencher) { + let mut rt = tokio::runtime::Builder::new() + .basic_scheduler() + .build() + .unwrap(); + + async fn task(lock: Arc>) { + let read = lock.read().await; + black_box(read); + } + + let lock = Arc::new(RwLock::new(())); + b.iter(|| { + let lock = lock.clone(); + rt.block_on(async move { + let write = lock.write().await; + tokio::join!{ + async move { drop(write) }, + task(lock.clone()), + task(lock.clone()), + task(lock.clone()), + task(lock.clone()), + task(lock.clone()), + }; + }) + }); +} + + +bencher::benchmark_group!( + sync_rwlock, + read_uncontended, + read_concurrent_uncontended, + read_concurrent_uncontended_multi, + read_concurrent_contended, + read_concurrent_contended_multi +); + +bencher::benchmark_main!(sync_rwlock); \ No newline at end of file From 034e14150a524f7cc99120b1b60262d72a9f43a8 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 17 Mar 2020 10:40:29 -0700 Subject: [PATCH 52/91] quick semaphore benches Signed-off-by: Eliza Weisman --- benches/Cargo.toml | 5 ++ benches/sync_semaphore.rs | 136 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 benches/sync_semaphore.rs diff --git a/benches/Cargo.toml b/benches/Cargo.toml index a02ee68f5a9..75723ce2105 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -28,3 +28,8 @@ harness = false name = "sync_rwlock" path = "sync_rwlock.rs" harness = false + +[[bench]] +name = "sync_semaphore" +path = "sync_semaphore.rs" +harness = false diff --git a/benches/sync_semaphore.rs b/benches/sync_semaphore.rs new file mode 100644 index 00000000000..ff772df1512 --- /dev/null +++ b/benches/sync_semaphore.rs @@ -0,0 +1,136 @@ +use bencher::{black_box, Bencher}; +use tokio::{task, sync::Semaphore}; +use std::sync::Arc; + +fn uncontended(b: &mut Bencher) { + let mut rt = tokio::runtime::Builder::new() + .core_threads(6) + .threaded_scheduler() + .build() + .unwrap(); + + let s = Arc::new(Semaphore::new(10)); + b.iter(|| { + + let s = s.clone(); + rt.block_on(async move { + for _ in 0..6 { + let permit = s.acquire().await; + drop(permit); + } + }) + }); +} + +async fn task(s: Arc) { + let permit = s.acquire().await; + drop(permit); +} + +fn uncontended_concurrent_multi(b: &mut Bencher) { + let mut rt = tokio::runtime::Builder::new() + .core_threads(6) + .threaded_scheduler() + .build() + .unwrap(); + + let s = Arc::new(Semaphore::new(10)); + b.iter(|| { + let s = s.clone(); + rt.block_on(async move { + let j = tokio::try_join!{ + task::spawn(task(s.clone())), + task::spawn(task(s.clone())), + task::spawn(task(s.clone())), + task::spawn(task(s.clone())), + task::spawn(task(s.clone())), + task::spawn(task(s.clone())) + }; + j.unwrap(); + + }) + }); +} + +fn uncontended_concurrent_single(b: &mut Bencher) { + let mut rt = tokio::runtime::Builder::new() + .basic_scheduler() + .build() + .unwrap(); + + let s = Arc::new(Semaphore::new(10)); + b.iter(|| { + let s = s.clone(); + rt.block_on(async move { + tokio::join!{ + task(s.clone()), + task(s.clone()), + task(s.clone()), + task(s.clone()), + task(s.clone()), + task(s.clone()) + }; + + }) + }); +} + + +fn contended_concurrent_multi(b: &mut Bencher) { + let mut rt = tokio::runtime::Builder::new() + .core_threads(6) + .threaded_scheduler() + .build() + .unwrap(); + + let s = Arc::new(Semaphore::new(5)); + b.iter(|| { + let s = s.clone(); + rt.block_on(async move { + let j = tokio::try_join!{ + task::spawn(task(s.clone())), + task::spawn(task(s.clone())), + task::spawn(task(s.clone())), + task::spawn(task(s.clone())), + task::spawn(task(s.clone())), + task::spawn(task(s.clone())) + }; + j.unwrap(); + + }) + }); +} + +fn contended_concurrent_single(b: &mut Bencher) { + let mut rt = tokio::runtime::Builder::new() + .basic_scheduler() + .build() + .unwrap(); + + let s = Arc::new(Semaphore::new(5)); + b.iter(|| { + let s = s.clone(); + rt.block_on(async move { + tokio::join!{ + task(s.clone()), + task(s.clone()), + task(s.clone()), + task(s.clone()), + task(s.clone()), + task(s.clone()) + }; + + }) + }); +} + +bencher::benchmark_group!( + sync_semaphore, + uncontended, + uncontended_concurrent_multi, + uncontended_concurrent_single, + contended_concurrent_multi, + contended_concurrent_single +); + +bencher::benchmark_main!(sync_semaphore); \ No newline at end of file From 0e0e13db7f1cc507b0b3fe48f3ecc515a895431e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 17 Mar 2020 12:37:36 -0700 Subject: [PATCH 53/91] add `LinkedList::split_back` method Signed-off-by: Eliza Weisman --- tokio/src/util/linked_list.rs | 87 +++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index fe6bfaeca9c..f3ebad628f6 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -118,6 +118,32 @@ impl LinkedList { } } + /// Splits this list off at `node`, returning a new list with `node` at its + /// front. + /// + /// If `node` is at the the front of this list, then this list will be empty after + /// splitting. If `node` is the last node in this list, then the returned + /// list will contain only `node`. + /// + /// # Safety + /// + /// The caller **must** ensure that `node` is currently contained by + /// `self` or not contained by any other list. + pub(crate) unsafe fn split_back(&mut self, node: NonNull) -> Self { + let new_tail = T::pointers(node).as_mut().prev.take().map(|prev| { + T::pointers(prev).as_mut().next = None; + prev + }); + if new_tail.is_none() { + self.head = None; + } + let tail = std::mem::replace(&mut self.tail, new_tail); + Self { + head: Some(node), + tail, + } + } + #[cfg(feature = "sync")] // used only by `batch_semaphore` pub(crate) fn last<'a>(&'a self) -> Option<&'a T::Target> { unsafe { Some(&*self.tail?.as_ptr()) } @@ -248,6 +274,7 @@ mod tests { use std::pin::Pin; + #[derive(Debug)] struct Entry { pointers: Pointers, val: i32, @@ -520,6 +547,66 @@ mod tests { assert!(i.next().is_none()); } + #[test] + fn split_back() { + let a = entry(1); + let b = entry(2); + let c = entry(3); + let d = entry(4); + + { + let mut list1 = LinkedList::<&Entry>::new(); + + push_all( + &mut list1, + &[a.as_ref(), b.as_ref(), c.as_ref(), d.as_ref()], + ); + let mut list2 = unsafe { list1.split_back(ptr(&a)) }; + + assert_eq!([2, 3, 4].to_vec(), collect_list(&mut list1)); + assert_eq!([1].to_vec(), collect_list(&mut list2)); + } + + { + let mut list1 = LinkedList::<&Entry>::new(); + + push_all( + &mut list1, + &[a.as_ref(), b.as_ref(), c.as_ref(), d.as_ref()], + ); + let mut list2 = unsafe { list1.split_back(ptr(&b)) }; + + assert_eq!([3, 4].to_vec(), collect_list(&mut list1)); + assert_eq!([1, 2].to_vec(), collect_list(&mut list2)); + } + + { + let mut list1 = LinkedList::<&Entry>::new(); + + push_all( + &mut list1, + &[a.as_ref(), b.as_ref(), c.as_ref(), d.as_ref()], + ); + let mut list2 = unsafe { list1.split_back(ptr(&c)) }; + + assert_eq!([4].to_vec(), collect_list(&mut list1)); + assert_eq!([1, 2, 3].to_vec(), collect_list(&mut list2)); + } + + { + let mut list1 = LinkedList::<&Entry>::new(); + + push_all( + &mut list1, + &[a.as_ref(), b.as_ref(), c.as_ref(), d.as_ref()], + ); + let mut list2 = unsafe { list1.split_back(ptr(&d)) }; + + assert_eq!(Vec::::new(), collect_list(&mut list1)); + assert_eq!([1, 2, 3, 4].to_vec(), collect_list(&mut list2)); + } + } + proptest::proptest! { #[test] fn fuzz_linked_list(ops: Vec) { From 705afe16e117a754cdb09710b80116c79b606e2a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 17 Mar 2020 12:54:20 -0700 Subject: [PATCH 54/91] add DoubleEndedIterator for linked_list::Iter~ Signed-off-by: Eliza Weisman --- tokio/src/util/linked_list.rs | 58 ++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index f3ebad628f6..548231e2cf1 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -210,34 +210,50 @@ impl LinkedList { // ===== impl Iter ===== -cfg_rt_threaded! { - use core::marker::PhantomData; +#[cfg(any(feature = "sync", feature = "rt-threaded"))] +pub(crate) struct Iter<'a, T: Link> { + curr: Option>, + #[cfg(feature = "sync")] + curr_back: Option>, + _p: core::marker::PhantomData<&'a T>, +} - pub(crate) struct Iter<'a, T: Link> { - curr: Option>, - _p: PhantomData<&'a T>, +#[cfg(any(feature = "sync", feature = "rt-threaded"))] +impl LinkedList { + pub(crate) fn iter(&self) -> Iter<'_, T> { + Iter { + curr: self.head, + #[cfg(feature = "sync")] + curr_back: self.tail, + _p: core::marker::PhantomData, + } } +} - impl LinkedList { - pub(crate) fn iter(&self) -> Iter<'_, T> { - Iter { - curr: self.head, - _p: PhantomData, - } - } +#[cfg(any(feature = "sync", feature = "rt-threaded"))] +impl<'a, T: Link> Iterator for Iter<'a, T> { + type Item = &'a T::Target; + + fn next(&mut self) -> Option<&'a T::Target> { + let curr = self.curr?; + // safety: the pointer references data contained by the list + self.curr = unsafe { T::pointers(curr).as_ref() }.next; + + // safety: the value is still owned by the linked list. + Some(unsafe { &*curr.as_ptr() }) } +} - impl<'a, T: Link> Iterator for Iter<'a, T> { - type Item = &'a T::Target; +#[cfg(feature = "sync")] +impl<'a, T: Link> DoubleEndedIterator for Iter<'a, T> { + fn next_back(&mut self) -> Option<&'a T::Target> { + let curr = self.curr_back?; - fn next(&mut self) -> Option<&'a T::Target> { - let curr = self.curr?; - // safety: the pointer references data contained by the list - self.curr = unsafe { T::pointers(curr).as_ref() }.next; + // safety: the pointer references data contained by the list + self.curr_back = unsafe { T::pointers(curr).as_ref() }.prev; - // safety: the value is still owned by the linked list. - Some(unsafe { &*curr.as_ptr() }) - } + // safety: the value is still owned by the linked list. + Some(unsafe { &*curr.as_ptr() }) } } From c9e48005530fb669a3c6af6333b1e840912c1bb3 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 17 Mar 2020 14:21:00 -0700 Subject: [PATCH 55/91] (WIP) move notification outside of lock Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 156 ++++++++++--------- tokio/src/sync/tests/loom_semaphore_batch.rs | 5 +- 2 files changed, 85 insertions(+), 76 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 8586cf0aa44..0fd21309904 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -116,6 +116,16 @@ struct Waiter { const CLOSED: usize = 1 << 17; +fn notify_all(list: &mut LinkedList) { + while let Some(waiter) = list.pop_back() { + let waker = unsafe { + waiter.as_ref().waker.with_mut(|waker| (*waker).take()) + }; + + waker.expect("if a node is in the wait list, it must have a waker").wake(); + } +} + impl Semaphore { /// Creates a new semaphore with the initial number of permits pub(crate) fn new(permits: usize) -> Self { @@ -153,20 +163,23 @@ impl Semaphore { return; } - // TODO: Handle overflow. A panic is not sufficient, the process must - // abort. - let prev = self.adding.fetch_add(added << 1, AcqRel); - if prev > 0 { - // Another thread is already assigning permits. It will continue to - // do so until `added` is drained, so we don't need to block on the - // lock. - return; - } + // // TODO: Handle overflow. A panic is not sufficient, the process must + // // abort. + // let prev = self.adding.fetch_add(added << 1, AcqRel); + // if prev > 0 { + // // Another thread is already assigning permits. It will continue to + // // do so until `added` is drained, so we don't need to block on the + // // lock. + // return; + // } // Otherwise, no other thread is assigning permits. Thus, we must lock // the semaphore's wait list and assign permits until `added` is // drained. - self.add_permits_locked(added, self.waiters.lock().unwrap(), false); + let mut notified = self.add_permits_locked(added, self.waiters.lock().unwrap()); + + notify_all(&mut notified); + } /// Closes the semaphore. This prevents the semaphore from issuing new @@ -179,81 +192,79 @@ impl Semaphore { // closing the semaphore. self.permits.fetch_or(CLOSED, Release); - // Acquire the `added`, setting the "closed" flag on the lock. - let prev = self.adding.fetch_or(1, AcqRel); - - if prev != 0 { - // Another thread has the lock and will be responsible for notifying - // pending waiters. - return; - } + // // Acquire the `added`, setting the "closed" flag on the lock. + // let prev = self.adding.fetch_or(1, AcqRel); + + // if prev != 0 { + // // Another thread has the lock and will be responsible for notifying + // // pending waiters. + // return; + // } + + { + let mut waiters = self.waiters.lock().unwrap(); + waiters.closed = true; + notify_all(&mut waiters.queue) + }; - let mut lock = self.waiters.lock().unwrap(); - lock.closed = true; - self.add_permits_locked(0, lock, true); } fn add_permits_locked( &self, mut rem: usize, mut waiters: MutexGuard<'_, Waitlist>, - mut closed: bool, - ) { + ) -> LinkedList { // The thread adding permits will loop until the counter of added // permits is drained. If threads add permits (or close the semaphore) // while we are holding the lock, we will add those permits as well, so // that the other thread does not need to block. - loop { - // How many permits are we releasing on this iteration? - let initial = rem; + // loop { + // // How many permits are we releasing on this iteration? + // let initial = rem; // Assign permits to the wait queue and notify any satisfied - // waiters. - while rem > 0 || waiters.closed { - let pop = match waiters.queue.last() { - Some(_) if waiters.closed => true, - Some(last) => last.assign_permits(&mut rem, waiters.closed), - None => { - self.permits.fetch_add(rem, AcqRel); - break; - } - }; - if pop { - let node = waiters.queue.pop_back().unwrap(); - // Safety: we are holding the lock on the wait queue, and - // thus have exclusive access to the nodes' wakers. - let waker = unsafe { - node.as_ref().waker.with_mut(|waker| (*waker).take()) - }; - - waker.expect("if a node is in the wait list, it must have a waker").wake(); - - } - } - - let n = initial << 1; - - let actual = if closed { - let actual = self.adding.fetch_sub(n | 1, AcqRel); - assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); - closed = false; - actual >> 1 + // waiters.\ + let mut last = None; + for waiter in waiters.queue.iter().rev() { + if waiter.assign_permits(&mut rem) { + last = Some(NonNull::from(waiter)); } else { - let actual = self.adding.fetch_sub(n, AcqRel); - assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); - if actual & 1 == 1 { - waiters.closed = true; - closed = true; - } - actual >> 1 - }; - - rem = actual - initial; - - if rem == 0 && !closed { break; } } + if rem > 0 { + self.permits.fetch_add(rem, Release); + } + if let Some(waiter) = last { + unsafe { + waiters.queue.split_back(waiter) + } + } else { + LinkedList::new() + } + // let n = initial << 1; + + // let actual = if closed { + // let actual = self.adding.fetch_sub(n | 1, AcqRel); + // assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); + // closed = false; + // actual >> 1 + // } else { + // let actual = self.adding.fetch_sub(n, AcqRel); + // assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); + // if actual & 1 == 1 { + // waiters.closed = true; + // closed = true; + // } + // actual >> 1 + // }; + + // rem = actual - initial; + + // if rem == 0 && !closed { + // break; + // } + // } } fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { @@ -527,11 +538,7 @@ impl Waiter { /// Assign permits to the waiter. /// /// Returns `true` if the waiter should be removed from the queue - fn assign_permits(&self, n: &mut usize, closed: bool) -> bool { - // If the wait list has closed, consume no permits but pop the node. - if closed { - return true; - } + fn assign_permits(&self, n: &mut usize) -> bool { let mut curr = self.state.load(Acquire); @@ -624,8 +631,7 @@ impl Drop for Acquire<'_> { // one to release these permits, but it's necessary to add them // since we will try to subtract them once we have finished // permit assignment. - self.semaphore.adding.fetch_add(acquired_permits << 1, AcqRel); - self.semaphore.add_permits_locked(acquired_permits, waiters, false); + self.semaphore.add_permits_locked(acquired_permits, waiters); } } else { diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index 4da45f344fb..66b9e0a2131 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -22,14 +22,17 @@ fn basic_usage() { async fn actor(shared: Arc) { let mut permit = Permit::new(); + dbg!("acquiring"); permit.acquire(1, &shared.semaphore).await.unwrap(); + dbg!("acquired"); let actual = shared.active.fetch_add(1, SeqCst); assert!(actual <= NUM - 1); let actual = shared.active.fetch_sub(1, SeqCst); assert!(actual <= NUM); - + dbg!("releasing"); permit.release(1, &shared.semaphore); + dbg!("released"); } loom::model(|| { From 389e231554b5b8199a872cdf8754a3f65c38c119 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 17 Mar 2020 15:07:00 -0700 Subject: [PATCH 56/91] clean up Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 114 ++++++------------------------ 1 file changed, 20 insertions(+), 94 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 0fd21309904..e68a11119aa 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -14,10 +14,6 @@ //! tasks are acquiring smaller numbers of permits. This means that in a //! use-case like tokio's read-write lock, writers will not be starved by //! readers. -//! -//! The linked list is guarded by a mutex, which must be acquired before -//! enqueueing or dequeueing a task. However, some operations are often -//! wait-free. use crate::loom::cell::CausalCell; use crate::loom::sync::{atomic::AtomicUsize, Mutex, MutexGuard}; use crate::util::linked_list::{self, LinkedList}; @@ -36,17 +32,6 @@ pub(crate) struct Semaphore { waiters: Mutex, /// The current number of available permits in the semaphore. permits: AtomicUsize, - - /// Permits in the process of being released. - /// - /// When releasing permits, if the lock on the semaphore's wait list is held - /// by another task, the task releasing permits adds them to this counter. - /// The task holding the lock will continue releasing permits until the - /// counter is drained, allowing the `release` operation to be wait free. - /// - /// The first bit of this counter indicates that the semaphore is closing. - /// Therefore, all values are shifted over one bit. - adding: AtomicUsize, } struct Waitlist { @@ -136,7 +121,6 @@ impl Semaphore { queue: LinkedList::new(), closed: false, }), - adding: AtomicUsize::new(0), } } @@ -148,36 +132,16 @@ impl Semaphore { /// Adds `n` new permits to the semaphore. pub(crate) fn add_permits(&self, added: usize) { - // Assigning permits to waiters requires locking the wait list, so that - // any waiters which receive their required number of permits can be - // dequeued and notified. However, if multiple threads attempt to add - // permits concurrently, we are able to make this operation wait-free. - // By tracking an atomic counter of permits added to the semaphore, we - // are able to determine if another thread is currently holding the lock - // to assign permits. If this is the case, we simply add our permits to - // the counter and return, so we don't need to acquire the lock. - // Otherwise, no other thread is adding permits, so we lock the wait and - // loop until the counter of added permits is drained. - if added == 0 { return; } - // // TODO: Handle overflow. A panic is not sufficient, the process must - // // abort. - // let prev = self.adding.fetch_add(added << 1, AcqRel); - // if prev > 0 { - // // Another thread is already assigning permits. It will continue to - // // do so until `added` is drained, so we don't need to block on the - // // lock. - // return; - // } - - // Otherwise, no other thread is assigning permits. Thus, we must lock - // the semaphore's wait list and assign permits until `added` is - // drained. + // Assign permits to the wait queue, returning a list containing all the + // waiters at the back of the queue that received enough permits to wake + // up. let mut notified = self.add_permits_locked(added, self.waiters.lock().unwrap()); - + + // Once we release the lock, notify all woken waiters. notify_all(&mut notified); } @@ -185,28 +149,11 @@ impl Semaphore { /// Closes the semaphore. This prevents the semaphore from issuing new /// permits and notifies all pending waiters. pub(crate) fn close(&self) { - // Closing the semaphore works similarly to adding permits: if another - // thread is already holding the lock on the wait list to assign - // permits, we simply set a bit in the counter of permits being added. - // The thread holding the lock will see that bit has been set, and begin - // closing the semaphore. - self.permits.fetch_or(CLOSED, Release); - // // Acquire the `added`, setting the "closed" flag on the lock. - // let prev = self.adding.fetch_or(1, AcqRel); - - // if prev != 0 { - // // Another thread has the lock and will be responsible for notifying - // // pending waiters. - // return; - // } - - { - let mut waiters = self.waiters.lock().unwrap(); - waiters.closed = true; - notify_all(&mut waiters.queue) - }; + let mut waiters = self.waiters.lock().unwrap(); + waiters.closed = true; + notify_all(&mut waiters.queue) } fn add_permits_locked( @@ -214,16 +161,8 @@ impl Semaphore { mut rem: usize, mut waiters: MutexGuard<'_, Waitlist>, ) -> LinkedList { - // The thread adding permits will loop until the counter of added - // permits is drained. If threads add permits (or close the semaphore) - // while we are holding the lock, we will add those permits as well, so - // that the other thread does not need to block. - - // loop { - // // How many permits are we releasing on this iteration? - // let initial = rem; - // Assign permits to the wait queue and notify any satisfied - // waiters.\ + // Starting from the back of the wait queue, assign each waiter as many + // permits as it needs until we run out of permits to assign. let mut last = None; for waiter in waiters.queue.iter().rev() { if waiter.assign_permits(&mut rem) { @@ -232,39 +171,26 @@ impl Semaphore { break; } } + + // If we assigned permits to all the waiters in the queue, and there are + // still permits left over, assign them back to the semaphore. if rem > 0 { self.permits.fetch_add(rem, Release); } + + // Split off the queue at the last waiter that was satisfied, creating a + // new list. Once we release the lock, we'll drain this list and notify + // the waiters in it. if let Some(waiter) = last { + // Safety: it's only safe to call `split_back` with a pointer to a + // node in the same list as the one we call `split_back` on. Since + // we got the waiter pointer from the list's iterator, this is fine. unsafe { waiters.queue.split_back(waiter) } } else { LinkedList::new() } - // let n = initial << 1; - - // let actual = if closed { - // let actual = self.adding.fetch_sub(n | 1, AcqRel); - // assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); - // closed = false; - // actual >> 1 - // } else { - // let actual = self.adding.fetch_sub(n, AcqRel); - // assert!(actual <= CLOSED, "permits to add overflowed! {}", actual); - // if actual & 1 == 1 { - // waiters.closed = true; - // closed = true; - // } - // actual >> 1 - // }; - - // rem = actual - initial; - - // if rem == 0 && !closed { - // break; - // } - // } } fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { From a18d569123436c74214e86f4b34ab9fc6d7a87ac Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 17 Mar 2020 15:33:15 -0700 Subject: [PATCH 57/91] fixup Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index e68a11119aa..a9d657b4f1a 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -343,8 +343,7 @@ impl Drop for Semaphore { impl fmt::Debug for Semaphore { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_struct("Semaphore") - .field("permits", &self.adding.load(Relaxed)) - .field("added", &self.adding.load(Relaxed)) + .field("permits", &self.permits.load(Relaxed)) .finish() } } @@ -553,11 +552,8 @@ impl Drop for Acquire<'_> { unsafe { waiters.queue.remove(node) }; if acquired_permits > 0 { - // we have already locked the mutex, so we know we will be the - // one to release these permits, but it's necessary to add them - // since we will try to subtract them once we have finished - // permit assignment. - self.semaphore.add_permits_locked(acquired_permits, waiters); + let mut notified = self.semaphore.add_permits_locked(acquired_permits, waiters); + notify_all(&mut notified); } } else { From f758fb6e6b46233c3e9b66e4036b7009e7ae25e1 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 18 Mar 2020 10:07:49 -0700 Subject: [PATCH 58/91] rm unused Signed-off-by: Eliza Weisman --- tokio/src/util/linked_list.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 548231e2cf1..8d2be133121 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -144,11 +144,6 @@ impl LinkedList { } } - #[cfg(feature = "sync")] // used only by `batch_semaphore` - pub(crate) fn last<'a>(&'a self) -> Option<&'a T::Target> { - unsafe { Some(&*self.tail?.as_ptr()) } - } - /// Returns whether the linked list doesn not contain any node pub(crate) fn is_empty(&self) -> bool { if self.head.is_some() { From 90f98c9b6e493c5dc2a0c4f70a34d787192eb5d0 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 18 Mar 2020 11:28:33 -0700 Subject: [PATCH 59/91] put back coop yielding in acquire futures this had to be moved around a little as we are no longer using `poll_fn` Signed-off-by: Eliza Weisman --- tokio/src/coop.rs | 63 +++++++++++++++++++++++++++++++++++++ tokio/src/sync/mutex.rs | 15 ++++++--- tokio/src/sync/rwlock.rs | 6 +++- tokio/src/sync/semaphore.rs | 7 ++++- 4 files changed, 84 insertions(+), 7 deletions(-) diff --git a/tokio/src/coop.rs b/tokio/src/coop.rs index e4cb0224160..7c2f181dc0d 100644 --- a/tokio/src/coop.rs +++ b/tokio/src/coop.rs @@ -46,6 +46,8 @@ // NOTE: The doctests in this module are ignored since the whole module is (currently) private. use std::cell::Cell; +use std::future::Future; +use std::pin::Pin; use std::task::{Context, Poll}; /// Constant used to determine how much "work" a task is allowed to do without yielding. @@ -250,6 +252,67 @@ pub async fn proceed() { poll_fn(|cx| poll_proceed(cx)).await; } +pin_project_lite::pin_project! { + /// A future that cooperatively yields to the task scheduler when polling, + /// if the task's budget is exhausted. + /// + /// Internally, this is simply a future combinator which calls + /// [`poll_proceed`] in its `poll` implementation before polling the wrapped + /// future. + /// + /// # Examples + /// + /// ```rust,ignore + /// # #[tokio::main] + /// # async fn main() { + /// use tokio::coop::CoopFutureExt; + /// + /// async { /* ... */ } + /// .cooperate() + /// .await; + /// # } + /// ``` + /// + /// [`poll_proceed`]: fn.poll_proceed.html + #[derive(Debug)] + pub(crate) struct CoopFuture { + #[pin] + future: F, + } +} + +impl Future for CoopFuture { + type Output = F::Output; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + ready!(poll_proceed(cx)); + self.project().future.poll(cx) + } +} + +impl CoopFuture { + /// Returns a new `CoopFuture` wrapping the given future. + fn new(future: F) -> Self { + Self { future } + } +} + +/// Extension trait providing `Future::cooperate` extension method. +/// +/// Note: if/when the co-op API becomes public, this method should probably be +/// provided by `FutureExt`, instead. +pub(crate) trait CoopFutureExt: Future { + /// Wrap `self` to cooperatively yield to the scheduler when polling, if the + /// task's budget is exhausted. + fn cooperate(self) -> CoopFuture + where + Self: Sized, + { + CoopFuture::new(self) + } +} + +impl CoopFutureExt for F where F: Future {} + #[cfg(all(test, not(loom)))] mod test { use super::*; diff --git a/tokio/src/sync/mutex.rs b/tokio/src/sync/mutex.rs index a0487a13197..7f3e0a4f177 100644 --- a/tokio/src/sync/mutex.rs +++ b/tokio/src/sync/mutex.rs @@ -78,6 +78,7 @@ //! //! [`Mutex`]: struct.Mutex.html //! [`MutexGuard`]: struct.MutexGuard.html +use crate::coop::CoopFutureExt; use crate::sync::batch_semaphore as semaphore; use std::cell::UnsafeCell; @@ -153,11 +154,15 @@ impl Mutex { /// A future that resolves on acquiring the lock and returns the `MutexGuard`. pub async fn lock(&self) -> MutexGuard<'_, T> { let mut permit = semaphore::Permit::new(); - permit.acquire(1, &self.s).await.unwrap_or_else(|_| { - // The semaphore was closed. but, we never explicitly close it, and we have a - // handle to it through the Arc, which means that this can never happen. - unreachable!() - }); + permit + .acquire(1, &self.s) + .cooperate() + .await + .unwrap_or_else(|_| { + // The semaphore was closed. but, we never explicitly close it, and we have a + // handle to it through the Arc, which means that this can never happen. + unreachable!() + }); MutexGuard { lock: self, permit } } diff --git a/tokio/src/sync/rwlock.rs b/tokio/src/sync/rwlock.rs index 2a08f9b84c3..e589b4edf73 100644 --- a/tokio/src/sync/rwlock.rs +++ b/tokio/src/sync/rwlock.rs @@ -1,3 +1,4 @@ +use crate::coop::CoopFutureExt; use crate::sync::batch_semaphore::{AcquireError, Permit, Semaphore}; use std::cell::UnsafeCell; use std::ops; @@ -113,7 +114,10 @@ struct ReleasingPermit<'a, T> { impl<'a, T> ReleasingPermit<'a, T> { async fn acquire(&mut self) -> Result<(), AcquireError> { - self.permit.acquire(self.num_permits, &self.lock.s).await + self.permit + .acquire(self.num_permits, &self.lock.s) + .cooperate() + .await } } diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index 62bf2e7bf39..d2ca37790d4 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -1,4 +1,5 @@ use super::batch_semaphore as ll; // low level implementation +use crate::coop::CoopFutureExt; /// Counting semaphore performing asynchronous permit aquisition. /// @@ -64,7 +65,11 @@ impl Semaphore { /// Acquires permit from the semaphore pub async fn acquire(&self) -> SemaphorePermit<'_> { let mut ll_permit = ll::Permit::new(); - ll_permit.acquire(1, &self.ll_sem).await.unwrap(); + ll_permit + .acquire(1, &self.ll_sem) + .cooperate() + .await + .unwrap(); SemaphorePermit { sem: &self, ll_permit, From 32a12bb17d8ab0634b83b8da2941769b71d3478a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 18 Mar 2020 11:57:40 -0700 Subject: [PATCH 60/91] (hopefully) fix feature flags Signed-off-by: Eliza Weisman --- tokio/src/coop.rs | 39 +++++++++++++++++++++-------------- tokio/src/util/linked_list.rs | 1 + 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/tokio/src/coop.rs b/tokio/src/coop.rs index 7c2f181dc0d..19302559350 100644 --- a/tokio/src/coop.rs +++ b/tokio/src/coop.rs @@ -275,7 +275,8 @@ pin_project_lite::pin_project! { /// /// [`poll_proceed`]: fn.poll_proceed.html #[derive(Debug)] - pub(crate) struct CoopFuture { + #[allow(unreachable_pub, dead_code)] + pub struct CoopFuture { #[pin] future: F, } @@ -291,27 +292,33 @@ impl Future for CoopFuture { impl CoopFuture { /// Returns a new `CoopFuture` wrapping the given future. - fn new(future: F) -> Self { + /// + #[allow(unreachable_pub, dead_code)] + pub fn new(future: F) -> Self { Self { future } } } -/// Extension trait providing `Future::cooperate` extension method. -/// -/// Note: if/when the co-op API becomes public, this method should probably be -/// provided by `FutureExt`, instead. -pub(crate) trait CoopFutureExt: Future { - /// Wrap `self` to cooperatively yield to the scheduler when polling, if the - /// task's budget is exhausted. - fn cooperate(self) -> CoopFuture - where - Self: Sized, - { - CoopFuture::new(self) +// Currently only used by `tokio::sync`; and if we make this combinator public, +// it should probably be on the `FutureExt` trait instead. +cfg_sync! { + /// Extension trait providing `Future::cooperate` extension method. + /// + /// Note: if/when the co-op API becomes public, this method should probably be + /// provided by `FutureExt`, instead. + pub(crate) trait CoopFutureExt: Future { + /// Wrap `self` to cooperatively yield to the scheduler when polling, if the + /// task's budget is exhausted. + fn cooperate(self) -> CoopFuture + where + Self: Sized, + { + CoopFuture::new(self) + } } -} -impl CoopFutureExt for F where F: Future {} + impl CoopFutureExt for F where F: Future {} +} #[cfg(all(test, not(loom)))] mod test { diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 8d2be133121..c54fa64b76f 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -129,6 +129,7 @@ impl LinkedList { /// /// The caller **must** ensure that `node` is currently contained by /// `self` or not contained by any other list. + #[cfg(feature = "sync")] pub(crate) unsafe fn split_back(&mut self, node: NonNull) -> Self { let new_tail = T::pointers(node).as_mut().prev.take().map(|prev| { T::pointers(prev).as_mut().next = None; From 5c76e6721c736284284fca1a973789f8e66cd4aa Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 19 Mar 2020 12:46:43 -0700 Subject: [PATCH 61/91] move waiter state inside of lock Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 167 +++++++++++++++--------------- 1 file changed, 86 insertions(+), 81 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index a9d657b4f1a..168301e9772 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -66,7 +66,7 @@ pub(crate) struct Acquire<'a> { enum PermitState { /// Currently waiting for permits to be made available and assigned to the /// waiter. - Waiting(u16), + Waiting(u16, u16), /// The number of acquired permits Acquired(u16), @@ -78,7 +78,7 @@ struct Waiter { /// /// This is either the number of remaining permits required by /// the waiter, or a flag indicating that the waiter is not yet queued. - state: AtomicUsize, + state: CausalCell, /// The waker to notify the task awaiting permits. /// @@ -223,28 +223,32 @@ impl Semaphore { cx: &mut Context<'_>, needed: u16, node: Pin<&mut Waiter>, + queued: bool, ) -> Poll> { - let mut state; let mut acquired = 0; + let (needed, mut lock) = if queued { + let lock = self.waiters.lock().unwrap(); + let needed = node.state.with(|curr| unsafe {*curr}); + (needed, Some(lock)) + } else { + (needed as usize, None) + }; // First, try to take the requested number of permits from the // semaphore. - let mut lock = None; let mut curr = self.permits.load(Acquire); let mut waiters = loop { - state = node.state.load(Acquire); - // Has the waiter already acquired all its needed permits? If so, - // we're done! - if state == 0 { - return Ready(Ok(())); - } + // state = node.state.load(Acquire); + // // Has the waiter already acquired all its needed permits? If so, + // // we're done! + // if state == 0 { + // return Ready(Ok(())); + // } // Has the semaphore closed? if curr & CLOSED > 0 { return Ready(Err(AcquireError::closed())); } - - let needed = cmp::min(state, needed as usize); let mut remaining = 0; let (next, acq) = if curr + acquired >= needed { let next = curr - (needed - acquired); @@ -263,6 +267,7 @@ impl Semaphore { // permits may have been released. Therefore, we will acquire a // new snapshot of the current state of the semaphore before // actually enqueueing the waiter.. + curr = self.permits.load(Acquire); continue; } @@ -274,7 +279,7 @@ impl Semaphore { ) { Ok(_) => { acquired += acq; - if remaining == 0 { + if remaining == 0 && !queued { return Ready(Ok(())); } break lock.unwrap(); @@ -284,36 +289,24 @@ impl Semaphore { drop(lock.take()); }; - if waiters.closed { - return Ready(Err(AcquireError(()))); - } + node.state.with_mut(|curr| { + let curr = unsafe { &mut *curr }; + if *curr & Waiter::UNQUEUED == Waiter::UNQUEUED { + *curr = needed; + } + }); - let next = match acquired.cmp(&state) { - // If the waiter is in the unqueued state, we need all the requested - // permits, minus the amount we just acquired. - _ if state >= Waiter::UNQUEUED => needed as usize - acquired, - // We have acquired all the needed permits! - cmp::Ordering::Equal => 0, - cmp::Ordering::Less => state - acquired, - cmp::Ordering::Greater => { - unreachable!("cannot have been assigned more than needed permits") + if dbg!(node.assign_permits(&mut acquired)) { + if dbg!(acquired) > 0 { + // We ended up with more permits than we needed. Release the + // back to the semaphore. + self.add_permits_locked(acquired, waiters); } - }; - - // Typically, one would probably expect to see a compare-and-swap loop - // here. However, in this case, we don't need to loop. Our most snapshot - // of the node's current state was acquired after we acquired the lock - // on the wait list. Because releasing permits to waiters also - // requires locking the wait list, the state cannot have changed since - // we acquired this snapshot. - node.state - .compare_exchange(state, next, AcqRel, Acquire) - .expect("state cannot have changed while the wait list was locked"); - - if next == 0 { return Ready(Ok(())); } + assert_eq!(acquired, 0); + // Otherwise, register the waker & enqueue the node. node.waker.with_mut(|waker| // Safety: the wait list is locked, so we may modify the waker. @@ -365,22 +358,27 @@ impl Permit { _ => false, } } - + /// Returns a future that tries to acquire the permit. If no permits are /// available, the current task is notified once a new permit becomes /// available. pub(crate) fn acquire<'a>( &'a mut self, - num_permits: u16, + mut num_permits: u16, semaphore: &'a Semaphore, ) -> Acquire<'a> { self.state = match self.state { - PermitState::Acquired(0) => PermitState::Waiting(num_permits), - PermitState::Waiting(n) => PermitState::Waiting(cmp::max(n, num_permits)), - PermitState::Acquired(n) => PermitState::Acquired(n), + PermitState::Acquired(0) => PermitState::Waiting(num_permits, 0), + PermitState::Waiting(n, n2) => { + num_permits = cmp::max(n, num_permits - n2); + PermitState::Waiting(num_permits, n2) + }, + PermitState::Acquired(n) => { + PermitState::Acquired(n) + }, }; Acquire { - node: Waiter::new(), + node: Waiter::new(num_permits), semaphore, permit: self, num_permits, @@ -396,7 +394,7 @@ impl Permit { use PermitState::*; match self.state { - Waiting(_) => unreachable!( + Waiting(_, _) => unreachable!( "if a permit is waiting, then it has been borrowed mutably by an `Acquire` future" ), Acquired(acquired) => { @@ -430,7 +428,7 @@ impl Permit { use PermitState::*; match self.state { - Waiting(_) => unreachable!( + Waiting(_, _) => unreachable!( "cannot forget permits while in wait queue; we are already borrowed mutably?" ), Acquired(acquired) => { @@ -451,10 +449,10 @@ impl Default for Permit { impl Waiter { const UNQUEUED: usize = 1 << 16; - fn new() -> Self { + fn new(num_permits: u16) -> Self { Waiter { waker: CausalCell::new(None), - state: AtomicUsize::new(Self::UNQUEUED), + state: CausalCell::new(num_permits as usize), pointers: linked_list::Pointers::new(), _p: PhantomPinned, } @@ -464,26 +462,16 @@ impl Waiter { /// /// Returns `true` if the waiter should be removed from the queue fn assign_permits(&self, n: &mut usize) -> bool { + self.state.with_mut(|curr| { + let curr = unsafe {&mut *curr}; - let mut curr = self.state.load(Acquire); - - loop { + dbg!(*n, *curr); // Assign up to `n` permits. - let assign = cmp::min(curr, *n); - let next = curr - assign; - match self - .state - .compare_exchange(curr, next, AcqRel, Acquire) - { - Ok(_) => { - *n = n.saturating_sub(assign); - - // If we have assigned all remaining permits, return true. - return next == 0; - } - Err(actual) => curr = actual, - } - } + let assign = cmp::min(*curr, *n); + *curr -= assign; + *n -= assign; + *curr == 0 + }) } } @@ -492,21 +480,36 @@ impl Future for Acquire<'_> { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let (node, semaphore, permit, needed) = self.project(); + let mut res = Pending; permit.state = match permit.state { PermitState::Acquired(n) if n >= needed => return Ready(Ok(())), PermitState::Acquired(n) => { - ready!(semaphore.poll_acquire(cx, needed - n, node))?; - PermitState::Acquired(needed) + match semaphore.poll_acquire(cx, needed - n, node, false) { + Ready(r) => { + r?; + res = Ready(Ok(())); + PermitState::Acquired(needed) + }, + Pending => { + PermitState::Waiting(needed, n) + }, + } } - PermitState::Waiting(n) => { - assert_eq!(n, needed, "permit cannot be modified while waiting"); - if node.state.load(Acquire) > 0 { - ready!(semaphore.poll_acquire(cx, needed, node))?; + PermitState::Waiting(n, n2) => { + assert_eq!(n + n2, needed, "permit cannot be modified while waiting"); + match semaphore.poll_acquire(cx, n - n2, node, true) { + Ready(r) => { + r?; + res = Ready(Ok(())); + PermitState::Acquired(needed) + }, + Pending => { + PermitState::Waiting(needed, n2) + }, } - PermitState::Acquired(needed) } }; - Ready(Ok(())) + res } } @@ -533,15 +536,17 @@ impl Acquire<'_> { impl Drop for Acquire<'_> { fn drop(&mut self) { - let state = self.node.state.load(Acquire); - // fast path: if we aren't actually waiting, no need to acquire the lock. - if state & Waiter::UNQUEUED == Waiter::UNQUEUED { - return; - } // This is where we ensure safety. The future is being dropped, // which means we must ensure that the waiter entry is no longer stored // in the linked list. let mut waiters = self.semaphore.waiters.lock().unwrap(); + let state = self.node.state.with(|curr| unsafe { *curr }); + + // are we even waiting? + if state & Waiter::UNQUEUED == Waiter::UNQUEUED { + return; + } + let node = NonNull::from(&mut self.node); if waiters.contains(node) { @@ -565,8 +570,8 @@ impl Drop for Acquire<'_> { // If the permit had transitioned to the `Waiting` state, put it back // into `Acquired`. - if let PermitState::Waiting(_) = self.permit.state { - self.permit.state = PermitState::Acquired(0); + if let PermitState::Waiting(_, prior) = self.permit.state { + self.permit.state = PermitState::Acquired(prior); } } } From 95bb327949d9497d784bdc6e0bcaacf2463fb6ac Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 19 Mar 2020 13:20:27 -0700 Subject: [PATCH 62/91] remove dbg Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 168301e9772..bc3d1c8b3c8 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -296,8 +296,8 @@ impl Semaphore { } }); - if dbg!(node.assign_permits(&mut acquired)) { - if dbg!(acquired) > 0 { + if node.assign_permits(&mut acquired) { + if acquired > 0 { // We ended up with more permits than we needed. Release the // back to the semaphore. self.add_permits_locked(acquired, waiters); @@ -465,7 +465,6 @@ impl Waiter { self.state.with_mut(|curr| { let curr = unsafe {&mut *curr}; - dbg!(*n, *curr); // Assign up to `n` permits. let assign = cmp::min(*curr, *n); *curr -= assign; From c9699dc827212c4f2aa5a8c05fd954abbb0c2bd2 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 19 Mar 2020 14:36:47 -0700 Subject: [PATCH 63/91] cleanup Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 205 +++++++++++++++--------------- 1 file changed, 105 insertions(+), 100 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index bc3d1c8b3c8..0901343dcb2 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -6,7 +6,7 @@ //! permits places its waker at the end of a queue. When new permits are made //! available (such as by releasing an initial acquisition), they are assigned //! to the task at the front of the queue, waking that task if its requested -//! number of permits is met. +//! number of permits is met. //! //! Because waiters are enqueued at the back of the linked list and dequeued //! from the front, the semaphore is fair. Tasks trying to acquire large numbers @@ -18,14 +18,14 @@ use crate::loom::cell::CausalCell; use crate::loom::sync::{atomic::AtomicUsize, Mutex, MutexGuard}; use crate::util::linked_list::{self, LinkedList}; -use std::{cmp, fmt}; use std::future::Future; use std::marker::PhantomPinned; use std::pin::Pin; use std::ptr::NonNull; use std::sync::atomic::Ordering::*; -use std::task::{Context, Poll, Waker}; use std::task::Poll::*; +use std::task::{Context, Poll, Waker}; +use std::{cmp, fmt}; /// An asynchronous counting semaphore which permits waiting on multiple permits at once. pub(crate) struct Semaphore { @@ -66,7 +66,12 @@ pub(crate) struct Acquire<'a> { enum PermitState { /// Currently waiting for permits to be made available and assigned to the /// waiter. - Waiting(u16, u16), + Waiting { + /// We are waiting to acquire this many permits. + needed: u16, + /// We have previously acquired this many permits. + acquired: u16, + }, /// The number of acquired permits Acquired(u16), @@ -74,24 +79,24 @@ enum PermitState { /// An entry in the wait queue. struct Waiter { - /// The current state of the waiter. - /// + /// The current state of the waiter. + /// /// This is either the number of remaining permits required by /// the waiter, or a flag indicating that the waiter is not yet queued. state: CausalCell, /// The waker to notify the task awaiting permits. - /// + /// /// # Safety - /// + /// /// This may only be accessed while the wait queue is locked. /// XXX: it would be nice if we could enforce this... waker: CausalCell>, /// Intrusive linked-list pointers. - /// + /// /// # Safety - /// + /// /// This may only be accessed while the wait queue is locked. pointers: linked_list::Pointers, @@ -103,11 +108,11 @@ const CLOSED: usize = 1 << 17; fn notify_all(list: &mut LinkedList) { while let Some(waiter) = list.pop_back() { - let waker = unsafe { - waiter.as_ref().waker.with_mut(|waker| (*waker).take()) - }; + let waker = unsafe { waiter.as_ref().waker.with_mut(|waker| (*waker).take()) }; - waker.expect("if a node is in the wait list, it must have a waker").wake(); + waker + .expect("if a node is in the wait list, it must have a waker") + .wake(); } } @@ -129,7 +134,6 @@ impl Semaphore { self.permits.load(SeqCst) & std::u16::MAX as usize } - /// Adds `n` new permits to the semaphore. pub(crate) fn add_permits(&self, added: usize) { if added == 0 { @@ -140,10 +144,9 @@ impl Semaphore { // waiters at the back of the queue that received enough permits to wake // up. let mut notified = self.add_permits_locked(added, self.waiters.lock().unwrap()); - + // Once we release the lock, notify all woken waiters. notify_all(&mut notified); - } /// Closes the semaphore. This prevents the semaphore from issuing new @@ -183,11 +186,9 @@ impl Semaphore { // the waiters in it. if let Some(waiter) = last { // Safety: it's only safe to call `split_back` with a pointer to a - // node in the same list as the one we call `split_back` on. Since + // node in the same list as the one we call `split_back` on. Since // we got the waiter pointer from the list's iterator, this is fine. - unsafe { - waiters.queue.split_back(waiter) - } + unsafe { waiters.queue.split_back(waiter) } } else { LinkedList::new() } @@ -208,10 +209,7 @@ impl Semaphore { let next = curr - num_permits as usize; - match self - .permits - .compare_exchange(curr, next, AcqRel, Acquire) - { + match self.permits.compare_exchange(curr, next, AcqRel, Acquire) { Ok(_) => return Ok(()), Err(actual) => curr = actual, } @@ -227,28 +225,27 @@ impl Semaphore { ) -> Poll> { let mut acquired = 0; + // If we are already in the wait queue, we need to lock the queue so we + // can access the wait queue entry's current state. let (needed, mut lock) = if queued { let lock = self.waiters.lock().unwrap(); - let needed = node.state.with(|curr| unsafe {*curr}); + // Safety: since we have acquired the lock, it is safe to look at + // the waiter's state. + let needed = node.state.with(|curr| unsafe { *curr }); (needed, Some(lock)) } else { (needed as usize, None) }; + // First, try to take the requested number of permits from the // semaphore. let mut curr = self.permits.load(Acquire); let mut waiters = loop { - // state = node.state.load(Acquire); - // // Has the waiter already acquired all its needed permits? If so, - // // we're done! - // if state == 0 { - // return Ready(Ok(())); - // } - // Has the semaphore closed? if curr & CLOSED > 0 { return Ready(Err(AcquireError::closed())); } + let mut remaining = 0; let (next, acq) = if curr + acquired >= needed { let next = curr - (needed - acquired); @@ -262,21 +259,25 @@ impl Semaphore { // No permits were immediately available, so this permit will // (probably) need to wait. We'll need to acquire a lock on the // wait queue. - lock = Some(self.waiters.lock().unwrap()); - // While we were waiting to lock the wait list, additional - // permits may have been released. Therefore, we will acquire a - // new snapshot of the current state of the semaphore before - // actually enqueueing the waiter.. - curr = self.permits.load(Acquire); - continue; + if let Ok(l) = self.waiters.try_lock() { + // If we were able to acquire the lock *now* without + // blocking, we don't need to reload the semaphore's permit state. + lock = Some(l) + } else { + lock = Some(self.waiters.lock().unwrap()); + // While we were waiting to lock the wait list, additional + // permits may have been released. Therefore, we will acquire a + // new snapshot of the current state of the semaphore before + // actually enqueueing the waiter.. + curr = self.permits.load(Acquire); + continue; + } } - match self.permits.compare_exchange_weak( - curr, - next, - AcqRel, - Acquire, - ) { + match self + .permits + .compare_exchange_weak(curr, next, AcqRel, Acquire) + { Ok(_) => { acquired += acq; if remaining == 0 && !queued { @@ -289,13 +290,6 @@ impl Semaphore { drop(lock.take()); }; - node.state.with_mut(|curr| { - let curr = unsafe { &mut *curr }; - if *curr & Waiter::UNQUEUED == Waiter::UNQUEUED { - *curr = needed; - } - }); - if node.assign_permits(&mut acquired) { if acquired > 0 { // We ended up with more permits than we needed. Release the @@ -308,12 +302,12 @@ impl Semaphore { assert_eq!(acquired, 0); // Otherwise, register the waker & enqueue the node. - node.waker.with_mut(|waker| - // Safety: the wait list is locked, so we may modify the waker. - unsafe { *waker = Some(cx.waker().clone()) } - ); - unsafe { + node.waker.with_mut(|waker| { + // Safety: the wait list is locked, so we may modify the waker. + *waker = Some(cx.waker().clone()); + }); + let node = Pin::into_inner_unchecked(node) as *mut _; let node = NonNull::new_unchecked(node); @@ -358,7 +352,7 @@ impl Permit { _ => false, } } - + /// Returns a future that tries to acquire the permit. If no permits are /// available, the current task is notified once a new permit becomes /// available. @@ -368,14 +362,24 @@ impl Permit { semaphore: &'a Semaphore, ) -> Acquire<'a> { self.state = match self.state { - PermitState::Acquired(0) => PermitState::Waiting(num_permits, 0), - PermitState::Waiting(n, n2) => { - num_permits = cmp::max(n, num_permits - n2); - PermitState::Waiting(num_permits, n2) - }, - PermitState::Acquired(n) => { - PermitState::Acquired(n) - }, + PermitState::Waiting { needed, acquired } => { + num_permits = cmp::max(needed, num_permits - acquired); + PermitState::Waiting { + needed: num_permits, + acquired, + } + } + PermitState::Acquired(acquired) => { + num_permits -= acquired; + if num_permits > 0 { + PermitState::Waiting { + needed: num_permits, + acquired, + } + } else { + PermitState::Acquired(acquired) + } + } }; Acquire { node: Waiter::new(num_permits), @@ -394,7 +398,7 @@ impl Permit { use PermitState::*; match self.state { - Waiting(_, _) => unreachable!( + Waiting { .. } => unreachable!( "if a permit is waiting, then it has been borrowed mutably by an `Acquire` future" ), Acquired(acquired) => { @@ -428,7 +432,7 @@ impl Permit { use PermitState::*; match self.state { - Waiting(_, _) => unreachable!( + Waiting { .. } => unreachable!( "cannot forget permits while in wait queue; we are already borrowed mutably?" ), Acquired(acquired) => { @@ -447,8 +451,6 @@ impl Default for Permit { } impl Waiter { - const UNQUEUED: usize = 1 << 16; - fn new(num_permits: u16) -> Self { Waiter { waker: CausalCell::new(None), @@ -463,7 +465,7 @@ impl Waiter { /// Returns `true` if the waiter should be removed from the queue fn assign_permits(&self, n: &mut usize) -> bool { self.state.with_mut(|curr| { - let curr = unsafe {&mut *curr}; + let curr = unsafe { &mut *curr }; // Assign up to `n` permits. let assign = cmp::min(*curr, *n); @@ -478,33 +480,38 @@ impl Future for Acquire<'_> { type Output = Result<(), AcquireError>; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + use PermitState::*; + let (node, semaphore, permit, needed) = self.project(); let mut res = Pending; permit.state = match permit.state { - PermitState::Acquired(n) if n >= needed => return Ready(Ok(())), - PermitState::Acquired(n) => { - match semaphore.poll_acquire(cx, needed - n, node, false) { - Ready(r) => { + Acquired(acquired) if acquired >= needed => return Ready(Ok(())), + Acquired(acquired) => { + match semaphore.poll_acquire(cx, needed - acquired, node, false) { + Ready(r) => { r?; res = Ready(Ok(())); PermitState::Acquired(needed) - }, - Pending => { - PermitState::Waiting(needed, n) - }, + } + Pending => PermitState::Waiting { needed, acquired }, } } - PermitState::Waiting(n, n2) => { - assert_eq!(n + n2, needed, "permit cannot be modified while waiting"); - match semaphore.poll_acquire(cx, n - n2, node, true) { - Ready(r) => { + PermitState::Waiting { + needed: n, + acquired, + } => { + assert_eq!( + n + acquired, + needed, + "permit cannot be modified while waiting" + ); + match semaphore.poll_acquire(cx, n - acquired, node, true) { + Ready(r) => { r?; res = Ready(Ok(())); PermitState::Acquired(needed) - }, - Pending => { - PermitState::Waiting(needed, n2) - }, + } + Pending => PermitState::Waiting { needed, acquired }, } } }; @@ -535,20 +542,20 @@ impl Acquire<'_> { impl Drop for Acquire<'_> { fn drop(&mut self) { + // If the future is completed, there is no node in the wait list, so we + // can skip acquiring the lock. + if let PermitState::Acquired(_) = self.permit.state { + return; + } + // This is where we ensure safety. The future is being dropped, // which means we must ensure that the waiter entry is no longer stored // in the linked list. let mut waiters = self.semaphore.waiters.lock().unwrap(); let state = self.node.state.with(|curr| unsafe { *curr }); - // are we even waiting? - if state & Waiter::UNQUEUED == Waiter::UNQUEUED { - return; - } - let node = NonNull::from(&mut self.node); if waiters.contains(node) { - let acquired_permits = self.num_permits as usize - state; // remove the entry from the list // @@ -559,18 +566,16 @@ impl Drop for Acquire<'_> { let mut notified = self.semaphore.add_permits_locked(acquired_permits, waiters); notify_all(&mut notified); } - } else { - // We don't need to continue holding the lock while modifying the + // We don't need to continue holding the lock while modifying the // permit's local state. drop(waiters); } - // If the permit had transitioned to the `Waiting` state, put it back // into `Acquired`. - if let PermitState::Waiting(_, prior) = self.permit.state { - self.permit.state = PermitState::Acquired(prior); + if let PermitState::Waiting { acquired, .. } = self.permit.state { + self.permit.state = PermitState::Acquired(acquired); } } } @@ -664,7 +669,7 @@ impl Waitlist { true } else if self.queue.is_first(&node) { debug_assert!( - self.queue.is_last(&node), + self.queue.is_last(&node), "if a node is unlinked but is the head of a queue, it must \ also be the tail of the queue" ); From da1b027c57e53626e308d96ce9b6bab26db4b10d Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 09:13:17 -0700 Subject: [PATCH 64/91] remove unneeded SeqCst Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 0901343dcb2..c556368787d 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -131,7 +131,7 @@ impl Semaphore { /// Returns the current number of available permits pub(crate) fn available_permits(&self) -> usize { - self.permits.load(SeqCst) & std::u16::MAX as usize + self.permits.load(Acquire) & std::u16::MAX as usize } /// Adds `n` new permits to the semaphore. From 819a59f2850405493b001baa1b8387ce2bcf3f91 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 09:13:42 -0700 Subject: [PATCH 65/91] notify outside of lock on close, too Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 13 ++++++++----- tokio/src/util/linked_list.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index c556368787d..999b7a3bfb5 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -106,7 +106,7 @@ struct Waiter { const CLOSED: usize = 1 << 17; -fn notify_all(list: &mut LinkedList) { +fn notify_all(list: LinkedList) { while let Some(waiter) = list.pop_back() { let waker = unsafe { waiter.as_ref().waker.with_mut(|waker| (*waker).take()) }; @@ -146,7 +146,7 @@ impl Semaphore { let mut notified = self.add_permits_locked(added, self.waiters.lock().unwrap()); // Once we release the lock, notify all woken waiters. - notify_all(&mut notified); + notify_all(notified); } /// Closes the semaphore. This prevents the semaphore from issuing new @@ -154,9 +154,12 @@ impl Semaphore { pub(crate) fn close(&self) { self.permits.fetch_or(CLOSED, Release); - let mut waiters = self.waiters.lock().unwrap(); - waiters.closed = true; - notify_all(&mut waiters.queue) + let mut notified = { + let mut waiters = self.waiters.lock().unwrap(); + waiters.closed = true; + waiters.queue.take_all() + }; + notify_all(notified) } fn add_permits_locked( diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index c54fa64b76f..4c427dcbf00 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -145,6 +145,17 @@ impl LinkedList { } } + /// Takes all entries from this list, returning a new list. + /// + /// This list will be left empty. + #[cfg(feature = "sync")] + pub(crate) fn take_all(&mut self) -> Self { + Self { + head: self.head.take(), + tail: self.tail.take(), + } + } + /// Returns whether the linked list doesn not contain any node pub(crate) fn is_empty(&self) -> bool { if self.head.is_some() { @@ -619,6 +630,26 @@ mod tests { } } + #[test] + fn take_all() { + let mut list1 = LinkedList::<&Entry>::new(); + let a = entry(1); + let b = entry(2); + + list1.push_front(a.as_ref()); + list1.push_front(b.as_ref()); + + assert!(!list1.is_empty()); + + let mut list2 = list1.take_all(); + + assert!(!list1.is_empty()); + assert!(list2.is_empty()); + + assert_eq!(Vec::::new(), collect_list(&mut list1)); + assert_eq!([1, 2].to_vec(), collect_list(&mut list2)); + } + proptest::proptest! { #[test] fn fuzz_linked_list(ops: Vec) { From 29fea220f1aa30079f353146bf8216be3b74e84d Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 09:17:54 -0700 Subject: [PATCH 66/91] carl-style feature flags Signed-off-by: Eliza Weisman --- tokio/src/util/linked_list.rs | 144 +++++++++++++++++----------------- 1 file changed, 74 insertions(+), 70 deletions(-) diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 4c427dcbf00..3ac1565ee8e 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -118,44 +118,6 @@ impl LinkedList { } } - /// Splits this list off at `node`, returning a new list with `node` at its - /// front. - /// - /// If `node` is at the the front of this list, then this list will be empty after - /// splitting. If `node` is the last node in this list, then the returned - /// list will contain only `node`. - /// - /// # Safety - /// - /// The caller **must** ensure that `node` is currently contained by - /// `self` or not contained by any other list. - #[cfg(feature = "sync")] - pub(crate) unsafe fn split_back(&mut self, node: NonNull) -> Self { - let new_tail = T::pointers(node).as_mut().prev.take().map(|prev| { - T::pointers(prev).as_mut().next = None; - prev - }); - if new_tail.is_none() { - self.head = None; - } - let tail = std::mem::replace(&mut self.tail, new_tail); - Self { - head: Some(node), - tail, - } - } - - /// Takes all entries from this list, returning a new list. - /// - /// This list will be left empty. - #[cfg(feature = "sync")] - pub(crate) fn take_all(&mut self) -> Self { - Self { - head: self.head.take(), - tail: self.tail.take(), - } - } - /// Returns whether the linked list doesn not contain any node pub(crate) fn is_empty(&self) -> bool { if self.head.is_some() { @@ -166,18 +128,6 @@ impl LinkedList { true } - /// Returns `true` if `node` is the first node in this list. - #[cfg(feature = "sync")] // used only by `batch_semaphore` - pub(crate) fn is_first(&self, node: &T::Handle) -> bool { - self.head == Some(T::as_raw(node)) - } - - /// Returns `true` if `node` is the last node in this list. - #[cfg(feature = "sync")] // used only by `batch_semaphore` - pub(crate) fn is_last(&self, node: &T::Handle) -> bool { - self.tail == Some(T::as_raw(node)) - } - /// Removes the specified node from the list /// /// # Safety @@ -215,6 +165,56 @@ impl LinkedList { } } +cfg_sync! { + impl LinkedList { + /// Returns `true` if `node` is the first node in this list. + pub(crate) fn is_first(&self, node: &T::Handle) -> bool { + self.head == Some(T::as_raw(node)) + } + + /// Returns `true` if `node` is the last node in this list. + pub(crate) fn is_last(&self, node: &T::Handle) -> bool { + self.tail == Some(T::as_raw(node)) + } + + /// Splits this list off at `node`, returning a new list with `node` at its + /// front. + /// + /// If `node` is at the the front of this list, then this list will be empty after + /// splitting. If `node` is the last node in this list, then the returned + /// list will contain only `node`. + /// + /// # Safety + /// + /// The caller **must** ensure that `node` is currently contained by + /// `self` or not contained by any other list. + pub(crate) unsafe fn split_back(&mut self, node: NonNull) -> Self { + let new_tail = T::pointers(node).as_mut().prev.take().map(|prev| { + T::pointers(prev).as_mut().next = None; + prev + }); + if new_tail.is_none() { + self.head = None; + } + let tail = std::mem::replace(&mut self.tail, new_tail); + Self { + head: Some(node), + tail, + } + } + + /// Takes all entries from this list, returning a new list. + /// + /// This list will be left empty. + pub(crate) fn take_all(&mut self) -> Self { + Self { + head: self.head.take(), + tail: self.tail.take(), + } + } + } +} + // ===== impl Iter ===== #[cfg(any(feature = "sync", feature = "rt-threaded"))] @@ -251,16 +251,17 @@ impl<'a, T: Link> Iterator for Iter<'a, T> { } } -#[cfg(feature = "sync")] -impl<'a, T: Link> DoubleEndedIterator for Iter<'a, T> { - fn next_back(&mut self) -> Option<&'a T::Target> { - let curr = self.curr_back?; +cfg_sync! { + impl<'a, T: Link> DoubleEndedIterator for Iter<'a, T> { + fn next_back(&mut self) -> Option<&'a T::Target> { + let curr = self.curr_back?; - // safety: the pointer references data contained by the list - self.curr_back = unsafe { T::pointers(curr).as_ref() }.prev; + // safety: the pointer references data contained by the list + self.curr_back = unsafe { T::pointers(curr).as_ref() }.prev; - // safety: the value is still owned by the linked list. - Some(unsafe { &*curr.as_ptr() }) + // safety: the value is still owned by the linked list. + Some(unsafe { &*curr.as_ptr() }) + } } } @@ -274,19 +275,22 @@ impl Pointers { next: None, } } +} - /// Returns `true` if this set of `Pointers` is linked to a previous or next - /// node. - /// - /// # Notes - /// - This does _not_ test for membership in a given list; simply - /// whether the pointers are null or not. - /// - If a node is the _only_ node in a list, calling `is_linked` on its - /// `Pointers` will return `false`, but `LinkedList::is_first` and - /// `LinkedList::is_last` will return `true`. - #[cfg(feature = "sync")] // used only by `batch_semaphore` - pub(crate) fn is_linked(&self) -> bool { - self.prev.is_some() || self.next.is_some() +cfg_sync! { + impl Pointers { + /// Returns `true` if this set of `Pointers` is linked to a previous or next + /// node. + /// + /// # Notes + /// - This does _not_ test for membership in a given list; simply + /// whether the pointers are null or not. + /// - If a node is the _only_ node in a list, calling `is_linked` on its + /// `Pointers` will return `false`, but `LinkedList::is_first` and + /// `LinkedList::is_last` will return `true`. + pub(crate) fn is_linked(&self) -> bool { + self.prev.is_some() || self.next.is_some() + } } } From bdce1578f39c32c7fb91fb3478710be3da013b4c Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 09:49:53 -0700 Subject: [PATCH 67/91] ag Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 999b7a3bfb5..76930ca61a2 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -567,7 +567,7 @@ impl Drop for Acquire<'_> { if acquired_permits > 0 { let mut notified = self.semaphore.add_permits_locked(acquired_permits, waiters); - notify_all(&mut notified); + notify_all(notified); } } else { // We don't need to continue holding the lock while modifying the From d9bf8b25438d0cc2cd9cf0d58f4d2020ba6cb17a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 12:30:58 -0700 Subject: [PATCH 68/91] simplify permit struct significantly Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 258 ++++++------------- tokio/src/sync/mutex.rs | 22 +- tokio/src/sync/rwlock.rs | 40 ++- tokio/src/sync/semaphore.rs | 12 +- tokio/src/sync/tests/loom_semaphore_batch.rs | 34 +-- tokio/src/sync/tests/semaphore_batch.rs | 147 ++++------- 6 files changed, 159 insertions(+), 354 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 76930ca61a2..b94bda6fdf1 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -51,30 +51,14 @@ pub(crate) struct AcquireError(()); #[derive(Debug)] pub(crate) struct Permit { - state: PermitState, + num_permits: u16, } pub(crate) struct Acquire<'a> { node: Waiter, semaphore: &'a Semaphore, - permit: &'a mut Permit, num_permits: u16, -} - -/// Permit state -#[derive(Debug, Copy, Clone)] -enum PermitState { - /// Currently waiting for permits to be made available and assigned to the - /// waiter. - Waiting { - /// We are waiting to acquire this many permits. - needed: u16, - /// We have previously acquired this many permits. - acquired: u16, - }, - - /// The number of acquired permits - Acquired(u16), + queued: bool, } /// An entry in the wait queue. @@ -106,7 +90,7 @@ struct Waiter { const CLOSED: usize = 1 << 17; -fn notify_all(list: LinkedList) { +fn notify_all(mut list: LinkedList) { while let Some(waiter) = list.pop_back() { let waker = unsafe { waiter.as_ref().waker.with_mut(|waker| (*waker).take()) }; @@ -143,7 +127,7 @@ impl Semaphore { // Assign permits to the wait queue, returning a list containing all the // waiters at the back of the queue that received enough permits to wake // up. - let mut notified = self.add_permits_locked(added, self.waiters.lock().unwrap()); + let notified = self.add_permits_locked(added, self.waiters.lock().unwrap()); // Once we release the lock, notify all woken waiters. notify_all(notified); @@ -154,7 +138,7 @@ impl Semaphore { pub(crate) fn close(&self) { self.permits.fetch_or(CLOSED, Release); - let mut notified = { + let notified = { let mut waiters = self.waiters.lock().unwrap(); waiters.closed = true; waiters.queue.take_all() @@ -162,6 +146,32 @@ impl Semaphore { notify_all(notified) } + pub(crate) fn try_acquire(&self, num_permits: u16) -> Result { + let mut curr = self.permits.load(Acquire); + loop { + // Has the semaphore closed? + if curr & CLOSED > 0 { + return Err(TryAcquireError::Closed); + } + + // Are there enough permits remaining? + if (curr as u16) < num_permits { + return Err(TryAcquireError::NoPermits); + } + + let next = curr - num_permits as usize; + + match self.permits.compare_exchange(curr, next, AcqRel, Acquire) { + Ok(_) => return Ok(Permit { num_permits }), + Err(actual) => curr = actual, + } + } + } + + pub(crate) fn acquire(&self, num_permits: u16) -> Acquire<'_> { + Acquire::new(self, num_permits) + } + fn add_permits_locked( &self, mut rem: usize, @@ -197,35 +207,13 @@ impl Semaphore { } } - fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { - let mut curr = self.permits.load(Acquire); - loop { - // Has the semaphore closed? - if curr & CLOSED > 0 { - return Err(TryAcquireError::Closed); - } - - // Are there enough permits remaining? - if (curr as u16) < num_permits { - return Err(TryAcquireError::NoPermits); - } - - let next = curr - num_permits as usize; - - match self.permits.compare_exchange(curr, next, AcqRel, Acquire) { - Ok(_) => return Ok(()), - Err(actual) => curr = actual, - } - } - } - fn poll_acquire( &self, cx: &mut Context<'_>, - needed: u16, + num_permits: u16, node: Pin<&mut Waiter>, queued: bool, - ) -> Poll> { + ) -> Poll> { let mut acquired = 0; // If we are already in the wait queue, we need to lock the queue so we @@ -237,7 +225,7 @@ impl Semaphore { let needed = node.state.with(|curr| unsafe { *curr }); (needed, Some(lock)) } else { - (needed as usize, None) + (num_permits as usize, None) }; // First, try to take the requested number of permits from the @@ -284,7 +272,7 @@ impl Semaphore { Ok(_) => { acquired += acq; if remaining == 0 && !queued { - return Ready(Ok(())); + return Ready(Ok(Permit { num_permits })); } break lock.unwrap(); } @@ -299,7 +287,7 @@ impl Semaphore { // back to the semaphore. self.add_permits_locked(acquired, waiters); } - return Ready(Ok(())); + return Ready(Ok(Permit { num_permits })); } assert_eq!(acquired, 0); @@ -339,57 +327,21 @@ impl fmt::Debug for Semaphore { } impl Permit { - /// Creates a new `Permit`. - /// - /// The permit begins in the "unacquired" state. - pub(crate) fn new() -> Permit { - use PermitState::Acquired; - - Permit { state: Acquired(0) } - } - - /// Returns `true` if the permit has been acquired - pub(crate) fn is_acquired(&self) -> bool { - match self.state { - PermitState::Acquired(num) if num > 0 => true, - _ => false, - } - } - /// Returns a future that tries to acquire the permit. If no permits are /// available, the current task is notified once a new permit becomes /// available. - pub(crate) fn acquire<'a>( - &'a mut self, - mut num_permits: u16, - semaphore: &'a Semaphore, - ) -> Acquire<'a> { - self.state = match self.state { - PermitState::Waiting { needed, acquired } => { - num_permits = cmp::max(needed, num_permits - acquired); - PermitState::Waiting { - needed: num_permits, - acquired, - } - } - PermitState::Acquired(acquired) => { - num_permits -= acquired; - if num_permits > 0 { - PermitState::Waiting { - needed: num_permits, - acquired, - } - } else { - PermitState::Acquired(acquired) - } - } - }; - Acquire { - node: Waiter::new(num_permits), - semaphore, - permit: self, - num_permits, + pub(crate) async fn acquire( + &mut self, + num_permits: u16, + semaphore: &Semaphore, + ) -> Result<(), AcquireError> { + if num_permits <= self.num_permits { + return Ok(()); } + let num_permits = num_permits - self.num_permits; + let acquired = semaphore.acquire(num_permits).await?; + self.num_permits += acquired.num_permits; + Ok(()) } /// Tries to acquire the permit. @@ -398,20 +350,12 @@ impl Permit { num_permits: u16, semaphore: &Semaphore, ) -> Result<(), TryAcquireError> { - use PermitState::*; - - match self.state { - Waiting { .. } => unreachable!( - "if a permit is waiting, then it has been borrowed mutably by an `Acquire` future" - ), - Acquired(acquired) => { - if acquired < num_permits { - semaphore.try_acquire(num_permits - acquired)?; - self.state = Acquired(num_permits); - } - } + if num_permits <= self.num_permits { + return Ok(()); } - + let num_permits = num_permits - self.num_permits; + let acquired = semaphore.try_acquire(num_permits)?; + self.num_permits += acquired.num_permits; Ok(()) } @@ -432,24 +376,9 @@ impl Permit { /// Will forget **at most** the number of acquired permits. This number is /// returned. pub(crate) fn forget(&mut self, n: u16) -> u16 { - use PermitState::*; - - match self.state { - Waiting { .. } => unreachable!( - "cannot forget permits while in wait queue; we are already borrowed mutably?" - ), - Acquired(acquired) => { - let n = cmp::min(n, acquired); - self.state = Acquired(acquired - n); - n - } - } - } -} - -impl Default for Permit { - fn default() -> Self { - Self::new() + let n = cmp::min(n, self.num_permits); + self.num_permits -= n; + n } } @@ -469,7 +398,6 @@ impl Waiter { fn assign_permits(&self, n: &mut usize) -> bool { self.state.with_mut(|curr| { let curr = unsafe { &mut *curr }; - // Assign up to `n` permits. let assign = cmp::min(*curr, *n); *curr -= assign; @@ -480,64 +408,48 @@ impl Waiter { } impl Future for Acquire<'_> { - type Output = Result<(), AcquireError>; + type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - use PermitState::*; - - let (node, semaphore, permit, needed) = self.project(); - let mut res = Pending; - permit.state = match permit.state { - Acquired(acquired) if acquired >= needed => return Ready(Ok(())), - Acquired(acquired) => { - match semaphore.poll_acquire(cx, needed - acquired, node, false) { - Ready(r) => { - r?; - res = Ready(Ok(())); - PermitState::Acquired(needed) - } - Pending => PermitState::Waiting { needed, acquired }, - } + let (node, semaphore, needed, queued) = self.project(); + match semaphore.poll_acquire(cx, needed, node, *queued) { + Pending => { + *queued = true; + Pending } - PermitState::Waiting { - needed: n, - acquired, - } => { - assert_eq!( - n + acquired, - needed, - "permit cannot be modified while waiting" - ); - match semaphore.poll_acquire(cx, n - acquired, node, true) { - Ready(r) => { - r?; - res = Ready(Ok(())); - PermitState::Acquired(needed) - } - Pending => PermitState::Waiting { needed, acquired }, - } + Ready(r) => { + *queued = false; + Ready(r) } - }; - res + } } } -impl Acquire<'_> { - fn project(self: Pin<&mut Self>) -> (Pin<&mut Waiter>, &Semaphore, &mut Permit, u16) { +impl<'a> Acquire<'a> { + fn new(semaphore: &'a Semaphore, num_permits: u16) -> Self { + Self { + node: Waiter::new(num_permits), + semaphore, + num_permits, + queued: false, + } + } + + fn project(self: Pin<&mut Self>) -> (Pin<&mut Waiter>, &Semaphore, u16, &mut bool) { fn is_unpin() {} unsafe { // Safety: all fields other than `node` are `Unpin` is_unpin::<&Semaphore>(); - is_unpin::<&mut Permit>(); + is_unpin::<&mut bool>(); is_unpin::(); let this = self.get_unchecked_mut(); ( Pin::new_unchecked(&mut this.node), &this.semaphore, - &mut this.permit, this.num_permits, + &mut this.queued, ) } } @@ -547,7 +459,7 @@ impl Drop for Acquire<'_> { fn drop(&mut self) { // If the future is completed, there is no node in the wait list, so we // can skip acquiring the lock. - if let PermitState::Acquired(_) = self.permit.state { + if !self.queued { return; } @@ -566,19 +478,9 @@ impl Drop for Acquire<'_> { unsafe { waiters.queue.remove(node) }; if acquired_permits > 0 { - let mut notified = self.semaphore.add_permits_locked(acquired_permits, waiters); + let notified = self.semaphore.add_permits_locked(acquired_permits, waiters); notify_all(notified); } - } else { - // We don't need to continue holding the lock while modifying the - // permit's local state. - drop(waiters); - } - - // If the permit had transitioned to the `Waiting` state, put it back - // into `Acquired`. - if let PermitState::Waiting { acquired, .. } = self.permit.state { - self.permit.state = PermitState::Acquired(acquired); } } } diff --git a/tokio/src/sync/mutex.rs b/tokio/src/sync/mutex.rs index 7f3e0a4f177..dd13cc305f3 100644 --- a/tokio/src/sync/mutex.rs +++ b/tokio/src/sync/mutex.rs @@ -153,24 +153,18 @@ impl Mutex { /// A future that resolves on acquiring the lock and returns the `MutexGuard`. pub async fn lock(&self) -> MutexGuard<'_, T> { - let mut permit = semaphore::Permit::new(); - permit - .acquire(1, &self.s) - .cooperate() - .await - .unwrap_or_else(|_| { - // The semaphore was closed. but, we never explicitly close it, and we have a - // handle to it through the Arc, which means that this can never happen. - unreachable!() - }); + let permit = self.s.acquire(1).cooperate().await.unwrap_or_else(|_| { + // The semaphore was closed. but, we never explicitly close it, and we have a + // handle to it through the Arc, which means that this can never happen. + unreachable!() + }); MutexGuard { lock: self, permit } } /// Tries to acquire the lock pub fn try_lock(&self) -> Result, TryLockError> { - let mut permit = semaphore::Permit::new(); - match permit.try_acquire(1, &self.s) { - Ok(_) => Ok(MutexGuard { lock: self, permit }), + match self.s.try_acquire(1) { + Ok(permit) => Ok(MutexGuard { lock: self, permit }), Err(_) => Err(TryLockError(())), } } @@ -205,14 +199,12 @@ where impl<'a, T> Deref for MutexGuard<'a, T> { type Target = T; fn deref(&self) -> &Self::Target { - assert!(self.permit.is_acquired()); unsafe { &*self.lock.c.get() } } } impl<'a, T> DerefMut for MutexGuard<'a, T> { fn deref_mut(&mut self) -> &mut Self::Target { - assert!(self.permit.is_acquired()); unsafe { &mut *self.lock.c.get() } } } diff --git a/tokio/src/sync/rwlock.rs b/tokio/src/sync/rwlock.rs index e589b4edf73..35644bba95b 100644 --- a/tokio/src/sync/rwlock.rs +++ b/tokio/src/sync/rwlock.rs @@ -113,11 +113,16 @@ struct ReleasingPermit<'a, T> { } impl<'a, T> ReleasingPermit<'a, T> { - async fn acquire(&mut self) -> Result<(), AcquireError> { - self.permit - .acquire(self.num_permits, &self.lock.s) - .cooperate() - .await + async fn acquire( + lock: &'a RwLock, + num_permits: u16, + ) -> Result, AcquireError> { + let permit = lock.s.acquire(num_permits).cooperate().await?; + Ok(Self { + permit, + num_permits, + lock, + }) } } @@ -201,12 +206,7 @@ impl RwLock { ///} /// ``` pub async fn read(&self) -> RwLockReadGuard<'_, T> { - let mut permit = ReleasingPermit { - num_permits: 1, - permit: Permit::new(), - lock: self, - }; - permit.acquire().await.unwrap_or_else(|_| { + let permit = ReleasingPermit::acquire(self, 1).await.unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and we have a // handle to it through the Arc, which means that this can never happen. unreachable!() @@ -237,17 +237,13 @@ impl RwLock { ///} /// ``` pub async fn write(&self) -> RwLockWriteGuard<'_, T> { - let mut permit = ReleasingPermit { - num_permits: MAX_READS as u16, - permit: Permit::new(), - lock: self, - }; - - permit.acquire().await.unwrap_or_else(|_| { - // The semaphore was closed. but, we never explicitly close it, and we have a - // handle to it through the Arc, which means that this can never happen. - unreachable!() - }); + let permit = ReleasingPermit::acquire(self, MAX_READS as u16) + .await + .unwrap_or_else(|_| { + // The semaphore was closed. but, we never explicitly close it, and we have a + // handle to it through the Arc, which means that this can never happen. + unreachable!() + }); RwLockWriteGuard { lock: self, permit } } diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index d2ca37790d4..cc66e4b26af 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -64,12 +64,7 @@ impl Semaphore { /// Acquires permit from the semaphore pub async fn acquire(&self) -> SemaphorePermit<'_> { - let mut ll_permit = ll::Permit::new(); - ll_permit - .acquire(1, &self.ll_sem) - .cooperate() - .await - .unwrap(); + let ll_permit = self.ll_sem.acquire(1).cooperate().await.unwrap(); SemaphorePermit { sem: &self, ll_permit, @@ -78,9 +73,8 @@ impl Semaphore { /// Tries to acquire a permit form the semaphore pub fn try_acquire(&self) -> Result, TryAcquireError> { - let mut ll_permit = ll::Permit::new(); - match ll_permit.try_acquire(1, &self.ll_sem) { - Ok(_) => Ok(SemaphorePermit { + match self.ll_sem.try_acquire(1) { + Ok(ll_permit) => Ok(SemaphorePermit { sem: self, ll_permit, }), diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index 66b9e0a2131..28eb1aa6b70 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -21,18 +21,13 @@ fn basic_usage() { } async fn actor(shared: Arc) { - let mut permit = Permit::new(); - dbg!("acquiring"); - permit.acquire(1, &shared.semaphore).await.unwrap(); - dbg!("acquired"); + let mut permit = shared.semaphore.acquire(1).await.unwrap(); let actual = shared.active.fetch_add(1, SeqCst); assert!(actual <= NUM - 1); let actual = shared.active.fetch_sub(1, SeqCst); assert!(actual <= NUM); - dbg!("releasing"); permit.release(1, &shared.semaphore); - dbg!("released"); } loom::model(|| { @@ -62,17 +57,13 @@ fn release() { { let semaphore = semaphore.clone(); thread::spawn(move || { - let mut permit = Permit::new(); - - block_on(permit.acquire(1, &semaphore)).unwrap(); + let mut permit = block_on(semaphore.acquire(1)).unwrap(); permit.release(1, &semaphore); }); } - let mut permit = Permit::new(); - - block_on(permit.acquire(1, &semaphore)).unwrap(); + let mut permit = block_on(semaphore.acquire(1)).unwrap(); permit.release(1, &semaphore); }); @@ -89,10 +80,8 @@ fn basic_closing() { let semaphore = semaphore.clone(); thread::spawn(move || { - let mut permit = Permit::new(); - for _ in 0..2 { - block_on(permit.acquire(1, &semaphore)).map_err(|_|())?; + let mut permit = block_on(semaphore.acquire(1)).map_err(|_|())?; permit.release(1, &semaphore); } @@ -116,9 +105,7 @@ fn concurrent_close() { let semaphore = semaphore.clone(); thread::spawn(move || { - let mut permit = Permit::new(); - - block_on(permit.acquire(1, &semaphore)).map_err(|_|())?; + let mut permit = block_on(semaphore.acquire(1)).map_err(|_|())?; permit.release(1, &semaphore); @@ -145,10 +132,8 @@ fn batch() { let active = active.clone(); ths.push(thread::spawn(move || { - let mut permit = Permit::new(); - for n in &[4, 10, 8] { - block_on(permit.acquire(*n, &semaphore)).unwrap(); + let mut permit = block_on(semaphore.acquire(*n)).unwrap(); active.fetch_add(*n as usize, SeqCst); @@ -176,13 +161,10 @@ fn batch() { fn release_during_acquire() { loom::model(|| { let semaphore = Arc::new(Semaphore::new(10)); - let mut permit1 = Permit::new(); - permit1.try_acquire(8, &semaphore).expect("try_acquire should succeed; semaphore uncontended"); + let mut permit1 = semaphore.try_acquire(8).expect("try_acquire should succeed; semaphore uncontended"); let semaphore2 = semaphore.clone(); let thread = thread::spawn(move || { - let mut permit = Permit::new(); - block_on(permit.acquire(4, &semaphore2)).unwrap(); - permit + block_on(semaphore2.acquire(4)).unwrap() }); permit1.release(8, &semaphore); diff --git a/tokio/src/sync/tests/semaphore_batch.rs b/tokio/src/sync/tests/semaphore_batch.rs index 8dc0993bff9..e0f301dd16d 100644 --- a/tokio/src/sync/tests/semaphore_batch.rs +++ b/tokio/src/sync/tests/semaphore_batch.rs @@ -1,4 +1,4 @@ -use crate::sync::batch_semaphore::{Permit, Semaphore}; +use crate::sync::batch_semaphore::Semaphore; use tokio_test::*; #[test] @@ -7,17 +7,12 @@ fn poll_acquire_one_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = Permit::new(); - assert!(!permit.is_acquired()); - - assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + let mut permit = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); assert_eq!(s.available_permits(), 99); - assert!(permit.is_acquired()); - // Polling again on the same waiter does not claim a new permit + // Polling again on the same permit does not claim a new permit assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); assert_eq!(s.available_permits(), 99); - assert!(permit.is_acquired()); } #[test] @@ -26,26 +21,19 @@ fn poll_acquire_many_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = Permit::new(); - assert!(!permit.is_acquired()); - - assert_ready_ok!(task::spawn(permit.acquire(5, &s)).poll()); + let mut permit = assert_ready_ok!(task::spawn(s.acquire(5)).poll()); assert_eq!(s.available_permits(), 95); - assert!(permit.is_acquired()); - // Polling again on the same waiter does not claim a new permit + // Polling again on the same permit does not claim a new permit assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); assert_eq!(s.available_permits(), 95); - assert!(permit.is_acquired()); assert_ready_ok!(task::spawn(permit.acquire(5, &s)).poll()); assert_eq!(s.available_permits(), 95); - assert!(permit.is_acquired()); // Polling for a larger number of permits acquires more assert_ready_ok!(task::spawn(permit.acquire(8, &s)).poll()); assert_eq!(s.available_permits(), 92); - assert!(permit.is_acquired()); } #[test] @@ -54,17 +42,12 @@ fn try_acquire_one_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = Permit::new(); - assert!(!permit.is_acquired()); - - assert_ok!(permit.try_acquire(1, &s)); + let mut permit = assert_ok!(s.try_acquire(1)); assert_eq!(s.available_permits(), 99); - assert!(permit.is_acquired()); - // Polling again on the same waiter does not claim a new permit + // Polling again on the same permit does not claim a new permit assert_ok!(permit.try_acquire(1, &s)); assert_eq!(s.available_permits(), 99); - assert!(permit.is_acquired()); } #[test] @@ -73,47 +56,32 @@ fn try_acquire_many_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = Permit::new(); - assert!(!permit.is_acquired()); - - assert_ok!(permit.try_acquire(5, &s)); + let mut permit = assert_ok!(s.try_acquire(5)); assert_eq!(s.available_permits(), 95); - assert!(permit.is_acquired()); - - // Polling again on the same waiter does not claim a new permit + // Polling again on the same permit does not claim a new permit assert_ok!(permit.try_acquire(5, &s)); assert_eq!(s.available_permits(), 95); - assert!(permit.is_acquired()); } #[test] - fn poll_acquire_one_unavailable() { let s = Semaphore::new(1); - let mut permit_1 = Permit::new(); - let mut permit_2 = Permit::new(); - - { - let mut acquire_1 = task::spawn(permit_1.acquire(1, &s)); - // Acquire the first permit - assert_ready_ok!(acquire_1.poll()); - assert_eq!(s.available_permits(), 0); - } + // Acquire the first permit + let mut permit_1 = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); + assert_eq!(s.available_permits(), 0); - { - let mut acquire_2 = task::spawn(permit_2.acquire(1, &s)); - // Try to acquire the second permit - assert_pending!(acquire_2.poll()); - assert_eq!(s.available_permits(), 0); + let mut acquire_2 = task::spawn(s.acquire(1)); + // Try to acquire the second permit + assert_pending!(acquire_2.poll()); + assert_eq!(s.available_permits(), 0); - permit_1.release(1, &s); + permit_1.release(1, &s); - assert_eq!(s.available_permits(), 0); - assert!(acquire_2.is_woken()); - assert_ready_ok!(acquire_2.poll()); - assert_eq!(s.available_permits(), 0); - } + assert_eq!(s.available_permits(), 0); + assert!(acquire_2.is_woken()); + let mut permit_2 = assert_ready_ok!(acquire_2.poll()); + assert_eq!(s.available_permits(), 0); assert_ready_ok!(task::spawn(permit_2.acquire(1, &s)).poll()); @@ -126,9 +94,7 @@ fn forget_acquired() { let s = Semaphore::new(1); // Polling for a permit succeeds immediately - let mut permit = Permit::new(); - - assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + let mut permit = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); assert_eq!(s.available_permits(), 0); @@ -140,21 +106,17 @@ fn forget_acquired() { fn poll_acquire_many_unavailable() { let s = Semaphore::new(5); - let mut permit_1 = Permit::new(); - let mut permit_2 = Permit::new(); - let mut permit_3 = Permit::new(); - // Acquire the first permit - assert_ready_ok!(task::spawn(permit_1.acquire(1, &s)).poll()); + let mut permit_1 = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); assert_eq!(s.available_permits(), 4); // Try to acquire the second permit - let mut acquire_2 = task::spawn(permit_2.acquire(5, &s)); + let mut acquire_2 = task::spawn(s.acquire(5)); assert_pending!(acquire_2.poll()); assert_eq!(s.available_permits(), 0); // Try to acquire the third permit - let mut acquire_3 = task::spawn(permit_3.acquire(3, &s)); + let mut acquire_3 = task::spawn(s.acquire(3)); assert_pending!(acquire_3.poll()); assert_eq!(s.available_permits(), 0); @@ -162,7 +124,7 @@ fn poll_acquire_many_unavailable() { assert_eq!(s.available_permits(), 0); assert!(acquire_2.is_woken()); - assert_ready_ok!(acquire_2.poll()); + let mut permit_2 = assert_ready_ok!(acquire_2.poll()); assert!(!acquire_3.is_woken()); assert_eq!(s.available_permits(), 0); @@ -182,19 +144,16 @@ fn poll_acquire_many_unavailable() { fn try_acquire_one_unavailable() { let s = Semaphore::new(1); - let mut permit_1 = Permit::new(); - let mut permit_2 = Permit::new(); - // Acquire the first permit - assert_ok!(permit_1.try_acquire(1, &s)); + let mut permit_1 = assert_ok!(s.try_acquire(1)); assert_eq!(s.available_permits(), 0); - assert_err!(permit_2.try_acquire(1, &s)); + assert_err!(s.try_acquire(1)); permit_1.release(1, &s); assert_eq!(s.available_permits(), 1); - assert_ok!(permit_2.try_acquire(1, &s)); + let mut permit_2 = assert_ok!(s.try_acquire(1)); permit_2.release(1, &s); assert_eq!(s.available_permits(), 1); @@ -204,19 +163,16 @@ fn try_acquire_one_unavailable() { fn try_acquire_many_unavailable() { let s = Semaphore::new(5); - let mut permit_1 = Permit::new(); - let mut permit_2 = Permit::new(); - // Acquire the first permit - assert_ok!(permit_1.try_acquire(1, &s)); + let mut permit_1 = assert_ok!(s.try_acquire(1)); assert_eq!(s.available_permits(), 4); - assert_err!(permit_2.try_acquire(5, &s)); + assert_err!(s.try_acquire(5)); permit_1.release(1, &s); assert_eq!(s.available_permits(), 5); - assert_ok!(permit_2.try_acquire(5, &s)); + let mut permit_2 = assert_ok!(s.try_acquire(5)); permit_2.release(1, &s); assert_eq!(s.available_permits(), 1); @@ -230,10 +186,8 @@ fn poll_acquire_one_zero_permits() { let s = Semaphore::new(0); assert_eq!(s.available_permits(), 0); - let mut permit = Permit::new(); - // Try to acquire the permit - let mut acquire = task::spawn(permit.acquire(1, &s)); + let mut acquire = task::spawn(s.acquire(1)); assert_pending!(acquire.poll()); s.add_permits(1); @@ -256,21 +210,17 @@ fn close_semaphore_prevents_acquire() { assert_eq!(5, s.available_permits()); - let mut permit_1 = Permit::new(); - let mut permit_2 = Permit::new(); - - assert_ready_err!(task::spawn(permit_1.acquire(1, &s)).poll()); + assert_ready_err!(task::spawn(s.acquire(1)).poll()); assert_eq!(5, s.available_permits()); - assert_ready_err!(task::spawn(permit_2.acquire(1, &s)).poll()); + assert_ready_err!(task::spawn(s.acquire(1)).poll()); assert_eq!(5, s.available_permits()); } #[test] fn close_semaphore_notifies_permit1() { let s = Semaphore::new(0); - let mut permit = Permit::new(); - let mut acquire = task::spawn(permit.acquire(1, &s)); + let mut acquire = task::spawn(s.acquire(1)); assert_pending!(acquire.poll()); @@ -284,17 +234,12 @@ fn close_semaphore_notifies_permit1() { fn close_semaphore_notifies_permit2() { let s = Semaphore::new(2); - let mut permit1 = Permit::new(); - let mut permit2 = Permit::new(); - let mut permit3 = Permit::new(); - let mut permit4 = Permit::new(); - // Acquire a couple of permits - assert_ready_ok!(task::spawn(permit1.acquire(1, &s)).poll()); - assert_ready_ok!(task::spawn(permit2.acquire(1, &s)).poll()); + let mut permit1 = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); + let mut permit2 = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); - let mut acquire3 = task::spawn(permit3.acquire(1, &s)); - let mut acquire4 = task::spawn(permit4.acquire(1, &s)); + let mut acquire3 = task::spawn(s.acquire(1)); + let mut acquire4 = task::spawn(s.acquire(1)); assert_pending!(acquire3.poll()); assert_pending!(acquire4.poll()); @@ -322,21 +267,15 @@ fn close_semaphore_notifies_permit2() { #[test] fn cancel_acquire_releases_permits() { let s = Semaphore::new(10); - let mut permit1 = Permit::new(); - permit1 - .try_acquire(4, &s) - .expect("uncontended try_acquire succeeds"); + let _permit1 = s.try_acquire(4).expect("uncontended try_acquire succeeds"); assert_eq!(6, s.available_permits()); - let mut permit2 = Permit::new(); - let mut acquire = task::spawn(permit2.acquire(8, &s)); + let mut acquire = task::spawn(s.acquire(8)); assert_pending!(acquire.poll()); assert_eq!(0, s.available_permits()); drop(acquire); assert_eq!(6, s.available_permits()); - permit2 - .try_acquire(6, &s) - .expect("should acquire successfully"); + assert_ok!(s.try_acquire(6)); } From 7406d54b2a55a32eef535aa14269f91973a8b877 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 13:22:54 -0700 Subject: [PATCH 69/91] simplify lock/cas loop interaction Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index b94bda6fdf1..d2799db981a 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -249,20 +249,12 @@ impl Semaphore { if remaining > 0 && lock.is_none() { // No permits were immediately available, so this permit will // (probably) need to wait. We'll need to acquire a lock on the - // wait queue. - if let Ok(l) = self.waiters.try_lock() { - // If we were able to acquire the lock *now* without - // blocking, we don't need to reload the semaphore's permit state. - lock = Some(l) - } else { - lock = Some(self.waiters.lock().unwrap()); - // While we were waiting to lock the wait list, additional - // permits may have been released. Therefore, we will acquire a - // new snapshot of the current state of the semaphore before - // actually enqueueing the waiter.. - curr = self.permits.load(Acquire); - continue; - } + // wait queue before continuing. We need to do this _before_ the + // CAS that sets the new value of the semaphore's `permits` + // counter. Otherwise, if we subtract the permits and then + // acquire the lock, we might miss additional permits being + // added while waiting for the lock. + lock = Some(self.waiters.lock().unwrap()); } match self From 1f8afa6c8c9aa425777aee3c227846936732f4f9 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 13:27:15 -0700 Subject: [PATCH 70/91] simplify lock/cas loop interaction, episode II Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index d2799db981a..a63a20ed3d3 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -270,7 +270,6 @@ impl Semaphore { } Err(actual) => curr = actual, } - drop(lock.take()); }; if node.assign_permits(&mut acquired) { From 2d8d413aaa4210419433418fa550c56ae46146dc Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 14:50:20 -0700 Subject: [PATCH 71/91] remove permit struct entirely Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 79 ++------------- tokio/src/sync/mutex.rs | 9 +- tokio/src/sync/rwlock.rs | 13 +-- tokio/src/sync/semaphore.rs | 17 ++-- tokio/src/sync/tests/loom_semaphore_batch.rs | 35 +++---- tokio/src/sync/tests/semaphore_batch.rs | 101 +++++++------------ 6 files changed, 76 insertions(+), 178 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index a63a20ed3d3..760dbe517df 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -39,21 +39,16 @@ struct Waitlist { closed: bool, } -/// Error returned by `Permit::try_acquire`. +/// Error returned by `Semaphore::try_acquire`. #[derive(Debug)] pub(crate) enum TryAcquireError { Closed, NoPermits, } -/// Error returned by `Permit::poll_acquire`. +/// Error returned by `Semaphore::acquire`. #[derive(Debug)] pub(crate) struct AcquireError(()); -#[derive(Debug)] -pub(crate) struct Permit { - num_permits: u16, -} - pub(crate) struct Acquire<'a> { node: Waiter, semaphore: &'a Semaphore, @@ -119,7 +114,7 @@ impl Semaphore { } /// Adds `n` new permits to the semaphore. - pub(crate) fn add_permits(&self, added: usize) { + pub(crate) fn release(&self, added: usize) { if added == 0 { return; } @@ -146,7 +141,7 @@ impl Semaphore { notify_all(notified) } - pub(crate) fn try_acquire(&self, num_permits: u16) -> Result { + pub(crate) fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { let mut curr = self.permits.load(Acquire); loop { // Has the semaphore closed? @@ -162,7 +157,7 @@ impl Semaphore { let next = curr - num_permits as usize; match self.permits.compare_exchange(curr, next, AcqRel, Acquire) { - Ok(_) => return Ok(Permit { num_permits }), + Ok(_) => return Ok(()), Err(actual) => curr = actual, } } @@ -213,7 +208,7 @@ impl Semaphore { num_permits: u16, node: Pin<&mut Waiter>, queued: bool, - ) -> Poll> { + ) -> Poll> { let mut acquired = 0; // If we are already in the wait queue, we need to lock the queue so we @@ -264,7 +259,7 @@ impl Semaphore { Ok(_) => { acquired += acq; if remaining == 0 && !queued { - return Ready(Ok(Permit { num_permits })); + return Ready(Ok(())); } break lock.unwrap(); } @@ -278,7 +273,7 @@ impl Semaphore { // back to the semaphore. self.add_permits_locked(acquired, waiters); } - return Ready(Ok(Permit { num_permits })); + return Ready(Ok(())); } assert_eq!(acquired, 0); @@ -317,62 +312,6 @@ impl fmt::Debug for Semaphore { } } -impl Permit { - /// Returns a future that tries to acquire the permit. If no permits are - /// available, the current task is notified once a new permit becomes - /// available. - pub(crate) async fn acquire( - &mut self, - num_permits: u16, - semaphore: &Semaphore, - ) -> Result<(), AcquireError> { - if num_permits <= self.num_permits { - return Ok(()); - } - let num_permits = num_permits - self.num_permits; - let acquired = semaphore.acquire(num_permits).await?; - self.num_permits += acquired.num_permits; - Ok(()) - } - - /// Tries to acquire the permit. - pub(crate) fn try_acquire( - &mut self, - num_permits: u16, - semaphore: &Semaphore, - ) -> Result<(), TryAcquireError> { - if num_permits <= self.num_permits { - return Ok(()); - } - let num_permits = num_permits - self.num_permits; - let acquired = semaphore.try_acquire(num_permits)?; - self.num_permits += acquired.num_permits; - Ok(()) - } - - /// Releases a permit back to the semaphore - pub(crate) fn release(&mut self, n: u16, semaphore: &Semaphore) { - let n = self.forget(n); - semaphore.add_permits(n as usize); - } - - /// Forgets the permit **without** releasing it back to the semaphore. - /// - /// After calling `forget`, `poll_acquire` is able to acquire new permit - /// from the sempahore. - /// - /// Repeatedly calling `forget` without associated calls to `add_permit` - /// will result in the semaphore losing all permits. - /// - /// Will forget **at most** the number of acquired permits. This number is - /// returned. - pub(crate) fn forget(&mut self, n: u16) -> u16 { - let n = cmp::min(n, self.num_permits); - self.num_permits -= n; - n - } -} - impl Waiter { fn new(num_permits: u16) -> Self { Waiter { @@ -399,7 +338,7 @@ impl Waiter { } impl Future for Acquire<'_> { - type Output = Result; + type Output = Result<(), AcquireError>; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let (node, semaphore, needed, queued) = self.project(); diff --git a/tokio/src/sync/mutex.rs b/tokio/src/sync/mutex.rs index dd13cc305f3..dac5ac16c42 100644 --- a/tokio/src/sync/mutex.rs +++ b/tokio/src/sync/mutex.rs @@ -107,7 +107,6 @@ pub struct Mutex { /// will succeed yet again. pub struct MutexGuard<'a, T> { lock: &'a Mutex, - permit: semaphore::Permit, } // As long as T: Send, it's fine to send and share Mutex between threads. @@ -153,18 +152,18 @@ impl Mutex { /// A future that resolves on acquiring the lock and returns the `MutexGuard`. pub async fn lock(&self) -> MutexGuard<'_, T> { - let permit = self.s.acquire(1).cooperate().await.unwrap_or_else(|_| { + self.s.acquire(1).cooperate().await.unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and we have a // handle to it through the Arc, which means that this can never happen. unreachable!() }); - MutexGuard { lock: self, permit } + MutexGuard { lock: self } } /// Tries to acquire the lock pub fn try_lock(&self) -> Result, TryLockError> { match self.s.try_acquire(1) { - Ok(permit) => Ok(MutexGuard { lock: self, permit }), + Ok(_) => Ok(MutexGuard { lock: self }), Err(_) => Err(TryLockError(())), } } @@ -177,7 +176,7 @@ impl Mutex { impl<'a, T> Drop for MutexGuard<'a, T> { fn drop(&mut self) { - self.permit.release(1, &self.lock.s); + self.lock.s.release(1) } } diff --git a/tokio/src/sync/rwlock.rs b/tokio/src/sync/rwlock.rs index 35644bba95b..bc5ab4b9498 100644 --- a/tokio/src/sync/rwlock.rs +++ b/tokio/src/sync/rwlock.rs @@ -1,5 +1,5 @@ use crate::coop::CoopFutureExt; -use crate::sync::batch_semaphore::{AcquireError, Permit, Semaphore}; +use crate::sync::batch_semaphore::{AcquireError, Semaphore}; use std::cell::UnsafeCell; use std::ops; @@ -108,7 +108,6 @@ pub struct RwLockWriteGuard<'a, T> { #[derive(Debug)] struct ReleasingPermit<'a, T> { num_permits: u16, - permit: Permit, lock: &'a RwLock, } @@ -117,18 +116,14 @@ impl<'a, T> ReleasingPermit<'a, T> { lock: &'a RwLock, num_permits: u16, ) -> Result, AcquireError> { - let permit = lock.s.acquire(num_permits).cooperate().await?; - Ok(Self { - permit, - num_permits, - lock, - }) + lock.s.acquire(num_permits).cooperate().await?; + Ok(Self { num_permits, lock }) } } impl<'a, T> Drop for ReleasingPermit<'a, T> { fn drop(&mut self) { - self.permit.release(self.num_permits, &self.lock.s); + self.lock.s.release(self.num_permits as usize); } } diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index cc66e4b26af..e34e49cc7fe 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -23,8 +23,7 @@ pub struct Semaphore { #[derive(Debug)] pub struct SemaphorePermit<'a> { sem: &'a Semaphore, - // the low level permit - ll_permit: ll::Permit, + permits: u16, } /// Error returned from the [`Semaphore::try_acquire`] function. @@ -59,24 +58,24 @@ impl Semaphore { /// Adds `n` new permits to the semaphore. pub fn add_permits(&self, n: usize) { - self.ll_sem.add_permits(n); + self.ll_sem.release(n); } /// Acquires permit from the semaphore pub async fn acquire(&self) -> SemaphorePermit<'_> { - let ll_permit = self.ll_sem.acquire(1).cooperate().await.unwrap(); + self.ll_sem.acquire(1).cooperate().await.unwrap(); SemaphorePermit { sem: &self, - ll_permit, + permits: 1, } } /// Tries to acquire a permit form the semaphore pub fn try_acquire(&self) -> Result, TryAcquireError> { match self.ll_sem.try_acquire(1) { - Ok(ll_permit) => Ok(SemaphorePermit { + Ok(_) => Ok(SemaphorePermit { sem: self, - ll_permit, + permits: 1, }), Err(_) => Err(TryAcquireError(())), } @@ -88,12 +87,12 @@ impl<'a> SemaphorePermit<'a> { /// This can be used to reduce the amount of permits available from a /// semaphore. pub fn forget(mut self) { - self.ll_permit.forget(1); + self.permits = 0; } } impl<'a> Drop for SemaphorePermit<'_> { fn drop(&mut self) { - self.ll_permit.release(1, &self.sem.ll_sem); + self.sem.add_permits(self.permits as usize); } } diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index 28eb1aa6b70..bdcfd86b8d7 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -21,13 +21,13 @@ fn basic_usage() { } async fn actor(shared: Arc) { - let mut permit = shared.semaphore.acquire(1).await.unwrap(); + shared.semaphore.acquire(1).await.unwrap(); let actual = shared.active.fetch_add(1, SeqCst); assert!(actual <= NUM - 1); let actual = shared.active.fetch_sub(1, SeqCst); assert!(actual <= NUM); - permit.release(1, &shared.semaphore); + shared.semaphore.release(1); } loom::model(|| { @@ -57,15 +57,14 @@ fn release() { { let semaphore = semaphore.clone(); thread::spawn(move || { - let mut permit = block_on(semaphore.acquire(1)).unwrap(); - - permit.release(1, &semaphore); + block_on(semaphore.acquire(1)).unwrap(); + semaphore.release(1); }); } - let mut permit = block_on(semaphore.acquire(1)).unwrap(); + block_on(semaphore.acquire(1)).unwrap(); - permit.release(1, &semaphore); + semaphore.release(1); }); } @@ -81,9 +80,9 @@ fn basic_closing() { thread::spawn(move || { for _ in 0..2 { - let mut permit = block_on(semaphore.acquire(1)).map_err(|_|())?; + block_on(semaphore.acquire(1)).map_err(|_|())?; - permit.release(1, &semaphore); + semaphore.release(1); } Ok::<(), ()>(()) @@ -105,10 +104,8 @@ fn concurrent_close() { let semaphore = semaphore.clone(); thread::spawn(move || { - let mut permit = block_on(semaphore.acquire(1)).map_err(|_|())?; - - permit.release(1, &semaphore); - + block_on(semaphore.acquire(1)).map_err(|_|())?; + semaphore.release(1); semaphore.close(); Ok::<(), ()>(()) @@ -133,7 +130,7 @@ fn batch() { ths.push(thread::spawn(move || { for n in &[4, 10, 8] { - let mut permit = block_on(semaphore.acquire(*n)).unwrap(); + block_on(semaphore.acquire(*n)).unwrap(); active.fetch_add(*n as usize, SeqCst); @@ -144,7 +141,7 @@ fn batch() { active.fetch_sub(*n as usize, SeqCst); - permit.release(*n, &semaphore); + semaphore.release(*n); } })); } @@ -161,15 +158,15 @@ fn batch() { fn release_during_acquire() { loom::model(|| { let semaphore = Arc::new(Semaphore::new(10)); - let mut permit1 = semaphore.try_acquire(8).expect("try_acquire should succeed; semaphore uncontended"); + lsemaphore.try_acquire(8).expect("try_acquire should succeed; semaphore uncontended"); let semaphore2 = semaphore.clone(); let thread = thread::spawn(move || { block_on(semaphore2.acquire(4)).unwrap() }); - permit1.release(8, &semaphore); - let mut permit2 = thread.join().unwrap(); - permit2.release(4, &semaphore); + semaphore.release(8); + thread.join().unwrap(); + semaphore.release(4); assert_eq!(10, semaphore.available_permits()); }) } diff --git a/tokio/src/sync/tests/semaphore_batch.rs b/tokio/src/sync/tests/semaphore_batch.rs index e0f301dd16d..60f3f231e76 100644 --- a/tokio/src/sync/tests/semaphore_batch.rs +++ b/tokio/src/sync/tests/semaphore_batch.rs @@ -7,11 +7,7 @@ fn poll_acquire_one_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); - assert_eq!(s.available_permits(), 99); - - // Polling again on the same permit does not claim a new permit - assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); + assert_ready_ok!(task::spawn(s.acquire(1)).poll()); assert_eq!(s.available_permits(), 99); } @@ -21,19 +17,11 @@ fn poll_acquire_many_available() { assert_eq!(s.available_permits(), 100); // Polling for a permit succeeds immediately - let mut permit = assert_ready_ok!(task::spawn(s.acquire(5)).poll()); - assert_eq!(s.available_permits(), 95); - - // Polling again on the same permit does not claim a new permit - assert_ready_ok!(task::spawn(permit.acquire(1, &s)).poll()); - assert_eq!(s.available_permits(), 95); - - assert_ready_ok!(task::spawn(permit.acquire(5, &s)).poll()); + assert_ready_ok!(task::spawn(s.acquire(5)).poll()); assert_eq!(s.available_permits(), 95); - // Polling for a larger number of permits acquires more - assert_ready_ok!(task::spawn(permit.acquire(8, &s)).poll()); - assert_eq!(s.available_permits(), 92); + assert_ready_ok!(task::spawn(s.acquire(5)).poll()); + assert_eq!(s.available_permits(), 90); } #[test] @@ -41,13 +29,11 @@ fn try_acquire_one_available() { let s = Semaphore::new(100); assert_eq!(s.available_permits(), 100); - // Polling for a permit succeeds immediately - let mut permit = assert_ok!(s.try_acquire(1)); + assert_ok!(s.try_acquire(1)); assert_eq!(s.available_permits(), 99); - // Polling again on the same permit does not claim a new permit - assert_ok!(permit.try_acquire(1, &s)); - assert_eq!(s.available_permits(), 99); + assert_ok!(s.try_acquire(1)); + assert_eq!(s.available_permits(), 98); } #[test] @@ -55,12 +41,11 @@ fn try_acquire_many_available() { let s = Semaphore::new(100); assert_eq!(s.available_permits(), 100); - // Polling for a permit succeeds immediately - let mut permit = assert_ok!(s.try_acquire(5)); - assert_eq!(s.available_permits(), 95); - // Polling again on the same permit does not claim a new permit - assert_ok!(permit.try_acquire(5, &s)); + assert_ok!(s.try_acquire(5)); assert_eq!(s.available_permits(), 95); + + assert_ok!(s.try_acquire(5)); + assert_eq!(s.available_permits(), 90); } #[test] @@ -68,7 +53,7 @@ fn poll_acquire_one_unavailable() { let s = Semaphore::new(1); // Acquire the first permit - let mut permit_1 = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); + assert_ready_ok!(task::spawn(s.acquire(1)).poll()); assert_eq!(s.available_permits(), 0); let mut acquire_2 = task::spawn(s.acquire(1)); @@ -76,38 +61,23 @@ fn poll_acquire_one_unavailable() { assert_pending!(acquire_2.poll()); assert_eq!(s.available_permits(), 0); - permit_1.release(1, &s); + s.release(1); assert_eq!(s.available_permits(), 0); assert!(acquire_2.is_woken()); - let mut permit_2 = assert_ready_ok!(acquire_2.poll()); + assert_ready_ok!(acquire_2.poll()); assert_eq!(s.available_permits(), 0); - assert_ready_ok!(task::spawn(permit_2.acquire(1, &s)).poll()); - - permit_2.release(1, &s); + s.release(1); assert_eq!(s.available_permits(), 1); } -#[test] -fn forget_acquired() { - let s = Semaphore::new(1); - - // Polling for a permit succeeds immediately - let mut permit = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); - - assert_eq!(s.available_permits(), 0); - - permit.forget(1); - assert_eq!(s.available_permits(), 0); -} - #[test] fn poll_acquire_many_unavailable() { let s = Semaphore::new(5); // Acquire the first permit - let mut permit_1 = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); + assert_ready_ok!(task::spawn(s.acquire(1)).poll()); assert_eq!(s.available_permits(), 4); // Try to acquire the second permit @@ -120,21 +90,20 @@ fn poll_acquire_many_unavailable() { assert_pending!(acquire_3.poll()); assert_eq!(s.available_permits(), 0); - permit_1.release(1, &s); + s.release(1); assert_eq!(s.available_permits(), 0); assert!(acquire_2.is_woken()); - let mut permit_2 = assert_ready_ok!(acquire_2.poll()); + assert_ready_ok!(acquire_2.poll()); assert!(!acquire_3.is_woken()); assert_eq!(s.available_permits(), 0); - drop(acquire_2); // drop the acquire future so we can now - permit_2.release(1, &s); + s.release(1); assert!(!acquire_3.is_woken()); assert_eq!(s.available_permits(), 0); - permit_2.release(2, &s); + s.release(2); assert!(acquire_3.is_woken()); assert_ready_ok!(acquire_3.poll()); @@ -145,17 +114,17 @@ fn try_acquire_one_unavailable() { let s = Semaphore::new(1); // Acquire the first permit - let mut permit_1 = assert_ok!(s.try_acquire(1)); + assert_ok!(s.try_acquire(1)); assert_eq!(s.available_permits(), 0); assert_err!(s.try_acquire(1)); - permit_1.release(1, &s); + s.release(1); assert_eq!(s.available_permits(), 1); - let mut permit_2 = assert_ok!(s.try_acquire(1)); + assert_ok!(s.try_acquire(1)); - permit_2.release(1, &s); + s.release(1); assert_eq!(s.available_permits(), 1); } @@ -164,20 +133,20 @@ fn try_acquire_many_unavailable() { let s = Semaphore::new(5); // Acquire the first permit - let mut permit_1 = assert_ok!(s.try_acquire(1)); + assert_ok!(s.try_acquire(1)); assert_eq!(s.available_permits(), 4); assert_err!(s.try_acquire(5)); - permit_1.release(1, &s); + s.release(1); assert_eq!(s.available_permits(), 5); - let mut permit_2 = assert_ok!(s.try_acquire(5)); + assert_ok!(s.try_acquire(5)); - permit_2.release(1, &s); + s.release(1); assert_eq!(s.available_permits(), 1); - permit_2.release(1, &s); + s.release(1); assert_eq!(s.available_permits(), 2); } @@ -190,7 +159,7 @@ fn poll_acquire_one_zero_permits() { let mut acquire = task::spawn(s.acquire(1)); assert_pending!(acquire.poll()); - s.add_permits(1); + s.release(1); assert!(acquire.is_woken()); assert_ready_ok!(acquire.poll()); @@ -235,8 +204,8 @@ fn close_semaphore_notifies_permit2() { let s = Semaphore::new(2); // Acquire a couple of permits - let mut permit1 = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); - let mut permit2 = assert_ready_ok!(task::spawn(s.acquire(1)).poll()); + assert_ready_ok!(task::spawn(s.acquire(1)).poll()); + assert_ready_ok!(task::spawn(s.acquire(1)).poll()); let mut acquire3 = task::spawn(s.acquire(1)); let mut acquire4 = task::spawn(s.acquire(1)); @@ -253,13 +222,13 @@ fn close_semaphore_notifies_permit2() { assert_eq!(0, s.available_permits()); - permit1.release(1, &s); + s.release(1); assert_eq!(1, s.available_permits()); - assert_ready_err!(task::spawn(permit1.acquire(1, &s)).poll()); + assert_ready_err!(task::spawn(s.acquire(1)).poll()); - permit2.release(1, &s); + s.release(1); assert_eq!(2, s.available_permits()); } From cc2afac49fd586385af407753b2b11dfd5a2d72f Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 14:54:15 -0700 Subject: [PATCH 72/91] unweaken cas Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 760dbe517df..6d715a724f6 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -252,10 +252,7 @@ impl Semaphore { lock = Some(self.waiters.lock().unwrap()); } - match self - .permits - .compare_exchange_weak(curr, next, AcqRel, Acquire) - { + match self.permits.compare_exchange(curr, next, AcqRel, Acquire) { Ok(_) => { acquired += acq; if remaining == 0 && !queued { From 5573c6569b5c16905e938a87fbed4f1fbc4f5280 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 15:05:03 -0700 Subject: [PATCH 73/91] fix backwards assertions in linked-list tests Signed-off-by: Eliza Weisman --- tokio/src/util/linked_list.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 3ac1565ee8e..b4cef69942f 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -647,8 +647,8 @@ mod tests { let mut list2 = list1.take_all(); - assert!(!list1.is_empty()); - assert!(list2.is_empty()); + assert!(list1.is_empty()); + assert!(!list2.is_empty()); assert_eq!(Vec::::new(), collect_list(&mut list1)); assert_eq!([1, 2].to_vec(), collect_list(&mut list2)); From e3c4b9fc69bc725a9f14e554f1925f6a8dee2e65 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 20 Mar 2020 17:23:07 -0700 Subject: [PATCH 74/91] rustfmt Signed-off-by: Eliza Weisman --- benches/sync_rwlock.rs | 17 ++++++----------- benches/sync_semaphore.rs | 20 +++++++------------- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/benches/sync_rwlock.rs b/benches/sync_rwlock.rs index c7b6395e1a3..4eca9807b2e 100644 --- a/benches/sync_rwlock.rs +++ b/benches/sync_rwlock.rs @@ -1,6 +1,6 @@ use bencher::{black_box, Bencher}; -use tokio::{task, sync::RwLock}; use std::sync::Arc; +use tokio::{sync::RwLock, task}; fn read_uncontended(b: &mut Bencher) { let mut rt = tokio::runtime::Builder::new() @@ -11,7 +11,6 @@ fn read_uncontended(b: &mut Bencher) { let lock = Arc::new(RwLock::new(())); b.iter(|| { - let lock = lock.clone(); rt.block_on(async move { for _ in 0..6 { @@ -38,7 +37,7 @@ fn read_concurrent_uncontended_multi(b: &mut Bencher) { b.iter(|| { let lock = lock.clone(); rt.block_on(async move { - let j = tokio::try_join!{ + let j = tokio::try_join! { task::spawn(task(lock.clone())), task::spawn(task(lock.clone())), task::spawn(task(lock.clone())), @@ -47,7 +46,6 @@ fn read_concurrent_uncontended_multi(b: &mut Bencher) { task::spawn(task(lock.clone())) }; j.unwrap(); - }) }); } @@ -67,7 +65,7 @@ fn read_concurrent_uncontended(b: &mut Bencher) { b.iter(|| { let lock = lock.clone(); rt.block_on(async move { - tokio::join!{ + tokio::join! { task(lock.clone()), task(lock.clone()), task(lock.clone()), @@ -75,7 +73,6 @@ fn read_concurrent_uncontended(b: &mut Bencher) { task(lock.clone()), task(lock.clone()) }; - }) }); } @@ -97,7 +94,7 @@ fn read_concurrent_contended_multi(b: &mut Bencher) { let lock = lock.clone(); rt.block_on(async move { let write = lock.write().await; - let j = tokio::try_join!{ + let j = tokio::try_join! { async move { drop(write); Ok(()) }, task::spawn(task(lock.clone())), task::spawn(task(lock.clone())), @@ -106,7 +103,6 @@ fn read_concurrent_contended_multi(b: &mut Bencher) { task::spawn(task(lock.clone())), }; j.unwrap(); - }) }); } @@ -127,7 +123,7 @@ fn read_concurrent_contended(b: &mut Bencher) { let lock = lock.clone(); rt.block_on(async move { let write = lock.write().await; - tokio::join!{ + tokio::join! { async move { drop(write) }, task(lock.clone()), task(lock.clone()), @@ -139,7 +135,6 @@ fn read_concurrent_contended(b: &mut Bencher) { }); } - bencher::benchmark_group!( sync_rwlock, read_uncontended, @@ -149,4 +144,4 @@ bencher::benchmark_group!( read_concurrent_contended_multi ); -bencher::benchmark_main!(sync_rwlock); \ No newline at end of file +bencher::benchmark_main!(sync_rwlock); diff --git a/benches/sync_semaphore.rs b/benches/sync_semaphore.rs index ff772df1512..c43311c0d35 100644 --- a/benches/sync_semaphore.rs +++ b/benches/sync_semaphore.rs @@ -1,6 +1,6 @@ -use bencher::{black_box, Bencher}; -use tokio::{task, sync::Semaphore}; +use bencher::Bencher; use std::sync::Arc; +use tokio::{sync::Semaphore, task}; fn uncontended(b: &mut Bencher) { let mut rt = tokio::runtime::Builder::new() @@ -11,7 +11,6 @@ fn uncontended(b: &mut Bencher) { let s = Arc::new(Semaphore::new(10)); b.iter(|| { - let s = s.clone(); rt.block_on(async move { for _ in 0..6 { @@ -38,7 +37,7 @@ fn uncontended_concurrent_multi(b: &mut Bencher) { b.iter(|| { let s = s.clone(); rt.block_on(async move { - let j = tokio::try_join!{ + let j = tokio::try_join! { task::spawn(task(s.clone())), task::spawn(task(s.clone())), task::spawn(task(s.clone())), @@ -47,7 +46,6 @@ fn uncontended_concurrent_multi(b: &mut Bencher) { task::spawn(task(s.clone())) }; j.unwrap(); - }) }); } @@ -62,7 +60,7 @@ fn uncontended_concurrent_single(b: &mut Bencher) { b.iter(|| { let s = s.clone(); rt.block_on(async move { - tokio::join!{ + tokio::join! { task(s.clone()), task(s.clone()), task(s.clone()), @@ -70,12 +68,10 @@ fn uncontended_concurrent_single(b: &mut Bencher) { task(s.clone()), task(s.clone()) }; - }) }); } - fn contended_concurrent_multi(b: &mut Bencher) { let mut rt = tokio::runtime::Builder::new() .core_threads(6) @@ -87,7 +83,7 @@ fn contended_concurrent_multi(b: &mut Bencher) { b.iter(|| { let s = s.clone(); rt.block_on(async move { - let j = tokio::try_join!{ + let j = tokio::try_join! { task::spawn(task(s.clone())), task::spawn(task(s.clone())), task::spawn(task(s.clone())), @@ -96,7 +92,6 @@ fn contended_concurrent_multi(b: &mut Bencher) { task::spawn(task(s.clone())) }; j.unwrap(); - }) }); } @@ -111,7 +106,7 @@ fn contended_concurrent_single(b: &mut Bencher) { b.iter(|| { let s = s.clone(); rt.block_on(async move { - tokio::join!{ + tokio::join! { task(s.clone()), task(s.clone()), task(s.clone()), @@ -119,7 +114,6 @@ fn contended_concurrent_single(b: &mut Bencher) { task(s.clone()), task(s.clone()) }; - }) }); } @@ -133,4 +127,4 @@ bencher::benchmark_group!( contended_concurrent_single ); -bencher::benchmark_main!(sync_semaphore); \ No newline at end of file +bencher::benchmark_main!(sync_semaphore); From 23257c105683a0571b03e66c3af8aa07e7a14dfc Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sat, 21 Mar 2020 09:49:19 -0700 Subject: [PATCH 75/91] fixup loom test Signed-off-by: Eliza Weisman --- tokio/src/sync/tests/loom_semaphore_batch.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index bdcfd86b8d7..48a3fbe1f72 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -141,7 +141,7 @@ fn batch() { active.fetch_sub(*n as usize, SeqCst); - semaphore.release(*n); + semaphore.release(*n as usize); } })); } @@ -158,7 +158,7 @@ fn batch() { fn release_during_acquire() { loom::model(|| { let semaphore = Arc::new(Semaphore::new(10)); - lsemaphore.try_acquire(8).expect("try_acquire should succeed; semaphore uncontended"); + semaphore.try_acquire(8).expect("try_acquire should succeed; semaphore uncontended"); let semaphore2 = semaphore.clone(); let thread = thread::spawn(move || { block_on(semaphore2.acquire(4)).unwrap() From 4de0d581858314f74c5b5a3a792e76d11a26b5ef Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sat, 21 Mar 2020 11:25:19 -0700 Subject: [PATCH 76/91] LinkedList fmt::Debug impls shouldn't need T: Debug Signed-off-by: Eliza Weisman --- tokio/src/util/linked_list.rs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index b4cef69942f..a3cbbae9f15 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -4,6 +4,7 @@ //! structure's APIs are `unsafe` as they require the caller to ensure the //! specified node is actually contained by the list. +use core::fmt; use core::mem::ManuallyDrop; use core::ptr::NonNull; @@ -11,7 +12,6 @@ use core::ptr::NonNull; /// /// Currently, the list is not emptied on drop. It is the caller's /// responsibility to ensure the list is empty before dropping it. -#[derive(Debug)] pub(crate) struct LinkedList { /// Linked list head head: Option>, @@ -53,7 +53,6 @@ pub(crate) unsafe trait Link { } /// Previous / next pointers -#[derive(Debug)] pub(crate) struct Pointers { /// The previous node in the list. null if there is no previous node. prev: Option>, @@ -215,6 +214,15 @@ cfg_sync! { } } +impl fmt::Debug for LinkedList { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LinkedList") + .field("head", &self.head) + .field("tail", &self.tail) + .finish() + } +} + // ===== impl Iter ===== #[cfg(any(feature = "sync", feature = "rt-threaded"))] @@ -294,6 +302,15 @@ cfg_sync! { } } +impl fmt::Debug for Pointers { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Pointers") + .field("prev", &self.prev) + .field("next", &self.next) + .finish() + } +} + #[cfg(test)] #[cfg(not(loom))] mod tests { From f22a18087034faef4f70c845af54626c8cac63b5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sat, 21 Mar 2020 13:02:30 -0700 Subject: [PATCH 77/91] fix racy close behavior Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 6d715a724f6..94c402a1fbc 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -131,10 +131,9 @@ impl Semaphore { /// Closes the semaphore. This prevents the semaphore from issuing new /// permits and notifies all pending waiters. pub(crate) fn close(&self) { - self.permits.fetch_or(CLOSED, Release); - let notified = { let mut waiters = self.waiters.lock().unwrap(); + self.permits.fetch_or(CLOSED, Release); waiters.closed = true; waiters.queue.take_all() }; From 9c9354bffabd3da7e7934ee407a4dd2c98bd14cc Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sat, 21 Mar 2020 13:25:31 -0700 Subject: [PATCH 78/91] run rustfmt on file cargo fmt doesn't see Signed-off-by: Eliza Weisman --- tokio/src/sync/tests/loom_semaphore_batch.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tokio/src/sync/tests/loom_semaphore_batch.rs b/tokio/src/sync/tests/loom_semaphore_batch.rs index 48a3fbe1f72..4c1936c5998 100644 --- a/tokio/src/sync/tests/loom_semaphore_batch.rs +++ b/tokio/src/sync/tests/loom_semaphore_batch.rs @@ -2,10 +2,10 @@ use crate::sync::batch_semaphore::*; use futures::future::poll_fn; use loom::future::block_on; +use loom::sync::atomic::AtomicUsize; use loom::thread; use std::future::Future; use std::pin::Pin; -use loom::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::SeqCst; use std::sync::Arc; use std::task::Poll::Ready; @@ -31,7 +31,6 @@ fn basic_usage() { } loom::model(|| { - let shared = Arc::new(Shared { semaphore: Semaphore::new(NUM), active: AtomicUsize::new(0), @@ -80,7 +79,7 @@ fn basic_closing() { thread::spawn(move || { for _ in 0..2 { - block_on(semaphore.acquire(1)).map_err(|_|())?; + block_on(semaphore.acquire(1)).map_err(|_| ())?; semaphore.release(1); } @@ -104,7 +103,7 @@ fn concurrent_close() { let semaphore = semaphore.clone(); thread::spawn(move || { - block_on(semaphore.acquire(1)).map_err(|_|())?; + block_on(semaphore.acquire(1)).map_err(|_| ())?; semaphore.release(1); semaphore.close(); @@ -158,11 +157,11 @@ fn batch() { fn release_during_acquire() { loom::model(|| { let semaphore = Arc::new(Semaphore::new(10)); - semaphore.try_acquire(8).expect("try_acquire should succeed; semaphore uncontended"); + semaphore + .try_acquire(8) + .expect("try_acquire should succeed; semaphore uncontended"); let semaphore2 = semaphore.clone(); - let thread = thread::spawn(move || { - block_on(semaphore2.acquire(4)).unwrap() - }); + let thread = thread::spawn(move || block_on(semaphore2.acquire(4)).unwrap()); semaphore.release(8); thread.join().unwrap(); From 2f1ead5373f4108636daa9e099fd10f90d73ce70 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sun, 22 Mar 2020 10:26:18 -0700 Subject: [PATCH 79/91] don't set unqueued when errored Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 94c402a1fbc..195d7b0878a 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -344,8 +344,9 @@ impl Future for Acquire<'_> { Pending } Ready(r) => { + r?; *queued = false; - Ready(r) + Ready(Ok(())) } } } From 077a61f7993c88f2fbed8846da7b916739b2a3cf Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sun, 22 Mar 2020 10:34:51 -0700 Subject: [PATCH 80/91] bring back atomic permit counter not strictly necessary, but it reduces unsafe... Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 51 ++++++++++++++++--------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 195d7b0878a..8f2ef296c0b 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -62,7 +62,7 @@ struct Waiter { /// /// This is either the number of remaining permits required by /// the waiter, or a flag indicating that the waiter is not yet queued. - state: CausalCell, + state: AtomicUsize, /// The waker to notify the task awaiting permits. /// @@ -212,16 +212,13 @@ impl Semaphore { // If we are already in the wait queue, we need to lock the queue so we // can access the wait queue entry's current state. - let (needed, mut lock) = if queued { - let lock = self.waiters.lock().unwrap(); - // Safety: since we have acquired the lock, it is safe to look at - // the waiter's state. - let needed = node.state.with(|curr| unsafe { *curr }); - (needed, Some(lock)) + let needed = if queued { + node.state.load(Acquire) } else { - (num_permits as usize, None) + num_permits as usize }; + let mut lock = None; // First, try to take the requested number of permits from the // semaphore. let mut curr = self.permits.load(Acquire); @@ -254,8 +251,12 @@ impl Semaphore { match self.permits.compare_exchange(curr, next, AcqRel, Acquire) { Ok(_) => { acquired += acq; - if remaining == 0 && !queued { - return Ready(Ok(())); + if remaining == 0 { + if !queued { + return Ready(Ok(())); + } else if lock.is_none() { + break self.waiters.lock().unwrap(); + } } break lock.unwrap(); } @@ -264,11 +265,7 @@ impl Semaphore { }; if node.assign_permits(&mut acquired) { - if acquired > 0 { - // We ended up with more permits than we needed. Release the - // back to the semaphore. - self.add_permits_locked(acquired, waiters); - } + self.add_permits_locked(acquired, waiters); return Ready(Ok(())); } @@ -312,7 +309,7 @@ impl Waiter { fn new(num_permits: u16) -> Self { Waiter { waker: CausalCell::new(None), - state: CausalCell::new(num_permits as usize), + state: AtomicUsize::new(num_permits as usize), pointers: linked_list::Pointers::new(), _p: PhantomPinned, } @@ -322,14 +319,18 @@ impl Waiter { /// /// Returns `true` if the waiter should be removed from the queue fn assign_permits(&self, n: &mut usize) -> bool { - self.state.with_mut(|curr| { - let curr = unsafe { &mut *curr }; - // Assign up to `n` permits. - let assign = cmp::min(*curr, *n); - *curr -= assign; - *n -= assign; - *curr == 0 - }) + let mut curr = self.state.load(Acquire); + loop { + let assign = cmp::min(curr, *n); + let next = curr - assign; + match self.state.compare_exchange(curr, next, AcqRel, Acquire) { + Ok(_) => { + *n -= assign; + return next == 0; + } + Err(actual) => curr = actual, + } + } } } @@ -394,7 +395,7 @@ impl Drop for Acquire<'_> { // which means we must ensure that the waiter entry is no longer stored // in the linked list. let mut waiters = self.semaphore.waiters.lock().unwrap(); - let state = self.node.state.with(|curr| unsafe { *curr }); + let state = self.node.state.load(Acquire); let node = NonNull::from(&mut self.node); if waiters.contains(node) { From be77a721f4c256688d8aebe32f23f376f4e9886f Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sun, 22 Mar 2020 13:35:09 -0700 Subject: [PATCH 81/91] simplify a few things Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 65 ++++++++++--------------------- tokio/src/util/linked_list.rs | 27 ------------- 2 files changed, 20 insertions(+), 72 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 8f2ef296c0b..3742aa48edc 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -130,6 +130,9 @@ impl Semaphore { /// Closes the semaphore. This prevents the semaphore from issuing new /// permits and notifies all pending waiters. + // This will be used once the bounded MPSC is updated to use the new + // semaphore implementation. + #[allow(dead_code)] pub(crate) fn close(&self) { let notified = { let mut waiters = self.waiters.lock().unwrap(); @@ -264,6 +267,10 @@ impl Semaphore { } }; + if waiters.closed { + return Ready(Err(AcquireError::closed())); + } + if node.assign_permits(&mut acquired) { self.add_permits_locked(acquired, waiters); return Ready(Ok(())); @@ -278,11 +285,11 @@ impl Semaphore { *waker = Some(cx.waker().clone()); }); - let node = Pin::into_inner_unchecked(node) as *mut _; - let node = NonNull::new_unchecked(node); - // If the waiter is not already in the wait queue, enqueue it. - if !waiters.contains(node) { + if !queued { + let node = Pin::into_inner_unchecked(node) as *mut _; + let node = NonNull::new_unchecked(node); + waiters.queue.push_front(node); } } @@ -291,12 +298,6 @@ impl Semaphore { } } -impl Drop for Semaphore { - fn drop(&mut self) { - self.close(); - } -} - impl fmt::Debug for Semaphore { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_struct("Semaphore") @@ -398,17 +399,15 @@ impl Drop for Acquire<'_> { let state = self.node.state.load(Acquire); let node = NonNull::from(&mut self.node); - if waiters.contains(node) { - let acquired_permits = self.num_permits as usize - state; - // remove the entry from the list - // - // Safety: we have locked the wait list. - unsafe { waiters.queue.remove(node) }; - - if acquired_permits > 0 { - let notified = self.semaphore.add_permits_locked(acquired_permits, waiters); - notify_all(notified); - } + let acquired_permits = self.num_permits as usize - state; + // remove the entry from the list + // + // Safety: we have locked the wait list. + unsafe { waiters.queue.remove(node) }; + + if acquired_permits > 0 { + let notified = self.semaphore.add_permits_locked(acquired_permits, waiters); + notify_all(notified); } } } @@ -488,27 +487,3 @@ unsafe impl linked_list::Link for Waiter { NonNull::from(&mut target.as_mut().pointers) } } - -impl Waitlist { - /// Returns true if this waitlist already contains the given waiter - fn contains(&self, node: NonNull) -> bool { - use linked_list::Link; - // Note: `is_linked` does not necessarily indicate that the node is - // linked with _this_ list. However, because nodes are only - // added/removed inside of `Acquire` futures, and a reference to the - // same `Semaphore` is present whenever the `Acquire` future calls this - // we know that the node cannot be linked with another list. - if unsafe { Waiter::pointers(node).as_ref() }.is_linked() { - true - } else if self.queue.is_first(&node) { - debug_assert!( - self.queue.is_last(&node), - "if a node is unlinked but is the head of a queue, it must \ - also be the tail of the queue" - ); - true - } else { - false - } - } -} diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index a3cbbae9f15..1a48803273a 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -166,16 +166,6 @@ impl LinkedList { cfg_sync! { impl LinkedList { - /// Returns `true` if `node` is the first node in this list. - pub(crate) fn is_first(&self, node: &T::Handle) -> bool { - self.head == Some(T::as_raw(node)) - } - - /// Returns `true` if `node` is the last node in this list. - pub(crate) fn is_last(&self, node: &T::Handle) -> bool { - self.tail == Some(T::as_raw(node)) - } - /// Splits this list off at `node`, returning a new list with `node` at its /// front. /// @@ -285,23 +275,6 @@ impl Pointers { } } -cfg_sync! { - impl Pointers { - /// Returns `true` if this set of `Pointers` is linked to a previous or next - /// node. - /// - /// # Notes - /// - This does _not_ test for membership in a given list; simply - /// whether the pointers are null or not. - /// - If a node is the _only_ node in a list, calling `is_linked` on its - /// `Pointers` will return `false`, but `LinkedList::is_first` and - /// `LinkedList::is_last` will return `true`. - pub(crate) fn is_linked(&self) -> bool { - self.prev.is_some() || self.next.is_some() - } - } -} - impl fmt::Debug for Pointers { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Pointers") From d20a93aba62e80b122ad614a684eaf40029ec2e6 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sun, 22 Mar 2020 13:38:06 -0700 Subject: [PATCH 82/91] expect instead of unwrap Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 3742aa48edc..e1a85e65d69 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -261,7 +261,7 @@ impl Semaphore { break self.waiters.lock().unwrap(); } } - break lock.unwrap(); + break lock.expect("lock must be acquired before waiting"); } Err(actual) => curr = actual, } From be8df9e5cdf0dbfb74c18416d3eb80300574a7da Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sun, 22 Mar 2020 13:47:49 -0700 Subject: [PATCH 83/91] ensure node pointers don't dangle on panics Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index e1a85e65d69..9305b31c9d6 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -395,7 +395,15 @@ impl Drop for Acquire<'_> { // This is where we ensure safety. The future is being dropped, // which means we must ensure that the waiter entry is no longer stored // in the linked list. - let mut waiters = self.semaphore.waiters.lock().unwrap(); + let mut waiters = match self.semaphore.waiters.lock() { + Ok(lock) => lock, + // Removing the node from the linked list is necessary to ensure + // safety. Even if the lock was poisoned, we need to make sure it is + // removed from the linked list before dropping it --- otherwise, + // the list will contain a dangling pointer to this node. + Err(e) => e.into_inner(), + }; + let state = self.node.state.load(Acquire); let node = NonNull::from(&mut self.node); From af8273f978f94b732115f1c30da8d20c2b9da6e8 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sun, 22 Mar 2020 13:52:18 -0700 Subject: [PATCH 84/91] cleanup/docs Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 9305b31c9d6..131355f59cd 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -136,6 +136,13 @@ impl Semaphore { pub(crate) fn close(&self) { let notified = { let mut waiters = self.waiters.lock().unwrap(); + // If the semaphore's permits counter has enough permits for an + // unqueued waiter to acquire all the permits it needs immediately, + // it won't touch the wait list. Therefore, we have to set a bit on + // the permit counter as well. However, we must do this while + // holding the lock --- otherwise, if we set the bit and then wait + // to acquire the lock we'll enter an inconsistent state where the + // permit counter is closed, but the wait list is not. self.permits.fetch_or(CLOSED, Release); waiters.closed = true; waiters.queue.take_all() @@ -178,11 +185,11 @@ impl Semaphore { // permits as it needs until we run out of permits to assign. let mut last = None; for waiter in waiters.queue.iter().rev() { - if waiter.assign_permits(&mut rem) { - last = Some(NonNull::from(waiter)); - } else { + // Was the waiter assigned enough permits to wake it? + if !waiter.assign_permits(&mut rem) { break; } + last = Some(NonNull::from(waiter)); } // If we assigned permits to all the waiters in the queue, and there are @@ -213,8 +220,6 @@ impl Semaphore { ) -> Poll> { let mut acquired = 0; - // If we are already in the wait queue, we need to lock the queue so we - // can access the wait queue entry's current state. let needed = if queued { node.state.load(Acquire) } else { @@ -404,15 +409,12 @@ impl Drop for Acquire<'_> { Err(e) => e.into_inner(), }; - let state = self.node.state.load(Acquire); - - let node = NonNull::from(&mut self.node); - let acquired_permits = self.num_permits as usize - state; // remove the entry from the list - // + let node = NonNull::from(&mut self.node); // Safety: we have locked the wait list. unsafe { waiters.queue.remove(node) }; + let acquired_permits = self.num_permits as usize - self.node.state.load(Acquire); if acquired_permits > 0 { let notified = self.semaphore.add_permits_locked(acquired_permits, waiters); notify_all(notified); From 4bb9fb1d8f832aec2697263b598ce23d98bd164e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 23 Mar 2020 09:34:24 -0700 Subject: [PATCH 85/91] fix max permits, use low bits for flags This undoes a potential breaking change where the max number of permits in semaphore a was reduced to `u16::MAX`. The max number of permits is now once again `usize::MAX >> 3`. In addition, flags are now stored in low bits rather than high bits, as is conventional throughout the rest of Tokio. This may decrease the risk of overflows. Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 32 ++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index 131355f59cd..d6553a38dca 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -83,7 +83,7 @@ struct Waiter { _p: PhantomPinned, } -const CLOSED: usize = 1 << 17; +const CLOSED: usize = 1; fn notify_all(mut list: LinkedList) { while let Some(waiter) = list.pop_back() { @@ -96,11 +96,20 @@ fn notify_all(mut list: LinkedList) { } impl Semaphore { + /// The maximum number of permits which a semaphore can hold. + /// + /// Note that this reserves three bits of flags in the permit counter, but + /// we only actually use one of them. However, the previous semaphore + /// implementation used three bits, so we will continue to reserve them to + /// avoid a breaking change if additional flags need to be aadded in the + /// future. + pub(crate) const MAX_PERMITS: usize = std::usize::MAX >> 3; + /// Creates a new semaphore with the initial number of permits pub(crate) fn new(permits: usize) -> Self { - assert!(permits <= std::u16::MAX as usize); + assert!(permits <= Self::MAX_PERMITS); Self { - permits: AtomicUsize::new(permits), + permits: AtomicUsize::new(permits << 1), waiters: Mutex::new(Waitlist { queue: LinkedList::new(), closed: false, @@ -110,7 +119,7 @@ impl Semaphore { /// Returns the current number of available permits pub(crate) fn available_permits(&self) -> usize { - self.permits.load(Acquire) & std::u16::MAX as usize + self.permits.load(Acquire) >> 1 } /// Adds `n` new permits to the semaphore. @@ -152,6 +161,7 @@ impl Semaphore { pub(crate) fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { let mut curr = self.permits.load(Acquire); + let num_permits = (num_permits as usize) << 1; loop { // Has the semaphore closed? if curr & CLOSED > 0 { @@ -159,11 +169,11 @@ impl Semaphore { } // Are there enough permits remaining? - if (curr as u16) < num_permits { + if curr < num_permits { return Err(TryAcquireError::NoPermits); } - let next = curr - num_permits as usize; + let next = curr - num_permits; match self.permits.compare_exchange(curr, next, AcqRel, Acquire) { Ok(_) => return Ok(()), @@ -195,7 +205,7 @@ impl Semaphore { // If we assigned permits to all the waiters in the queue, and there are // still permits left over, assign them back to the semaphore. if rem > 0 { - self.permits.fetch_add(rem, Release); + self.permits.fetch_add(rem << 1, Release); } // Split off the queue at the last waiter that was satisfied, creating a @@ -221,9 +231,9 @@ impl Semaphore { let mut acquired = 0; let needed = if queued { - node.state.load(Acquire) + node.state.load(Acquire) << 1 } else { - num_permits as usize + (num_permits as usize) << 1 }; let mut lock = None; @@ -239,10 +249,10 @@ impl Semaphore { let mut remaining = 0; let (next, acq) = if curr + acquired >= needed { let next = curr - (needed - acquired); - (next, needed) + (next, needed >> 1) } else { remaining = (needed - acquired) - curr; - (0, curr) + (0, curr >> 1) }; if remaining > 0 && lock.is_none() { From 67fe42081018303cb348608b11cc489940e05625 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 23 Mar 2020 09:38:32 -0700 Subject: [PATCH 86/91] const-ify permit shift amount Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index d6553a38dca..c1b6b119236 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -83,8 +83,6 @@ struct Waiter { _p: PhantomPinned, } -const CLOSED: usize = 1; - fn notify_all(mut list: LinkedList) { while let Some(waiter) = list.pop_back() { let waker = unsafe { waiter.as_ref().waker.with_mut(|waker| (*waker).take()) }; @@ -104,12 +102,14 @@ impl Semaphore { /// avoid a breaking change if additional flags need to be aadded in the /// future. pub(crate) const MAX_PERMITS: usize = std::usize::MAX >> 3; + const CLOSED: usize = 1; + const PERMIT_SHIFT: usize = 1; /// Creates a new semaphore with the initial number of permits pub(crate) fn new(permits: usize) -> Self { assert!(permits <= Self::MAX_PERMITS); Self { - permits: AtomicUsize::new(permits << 1), + permits: AtomicUsize::new(permits << Self::PERMIT_SHIFT), waiters: Mutex::new(Waitlist { queue: LinkedList::new(), closed: false, @@ -119,7 +119,7 @@ impl Semaphore { /// Returns the current number of available permits pub(crate) fn available_permits(&self) -> usize { - self.permits.load(Acquire) >> 1 + self.permits.load(Acquire) >> Self::PERMIT_SHIFT } /// Adds `n` new permits to the semaphore. @@ -152,7 +152,7 @@ impl Semaphore { // holding the lock --- otherwise, if we set the bit and then wait // to acquire the lock we'll enter an inconsistent state where the // permit counter is closed, but the wait list is not. - self.permits.fetch_or(CLOSED, Release); + self.permits.fetch_or(Self::CLOSED, Release); waiters.closed = true; waiters.queue.take_all() }; @@ -161,10 +161,10 @@ impl Semaphore { pub(crate) fn try_acquire(&self, num_permits: u16) -> Result<(), TryAcquireError> { let mut curr = self.permits.load(Acquire); - let num_permits = (num_permits as usize) << 1; + let num_permits = (num_permits as usize) << Self::PERMIT_SHIFT; loop { // Has the semaphore closed? - if curr & CLOSED > 0 { + if curr & Self::CLOSED > 0 { return Err(TryAcquireError::Closed); } @@ -205,7 +205,7 @@ impl Semaphore { // If we assigned permits to all the waiters in the queue, and there are // still permits left over, assign them back to the semaphore. if rem > 0 { - self.permits.fetch_add(rem << 1, Release); + self.permits.fetch_add(rem << Self::PERMIT_SHIFT, Release); } // Split off the queue at the last waiter that was satisfied, creating a @@ -231,9 +231,9 @@ impl Semaphore { let mut acquired = 0; let needed = if queued { - node.state.load(Acquire) << 1 + node.state.load(Acquire) << Self::PERMIT_SHIFT } else { - (num_permits as usize) << 1 + (num_permits as usize) << Self::PERMIT_SHIFT }; let mut lock = None; @@ -242,17 +242,17 @@ impl Semaphore { let mut curr = self.permits.load(Acquire); let mut waiters = loop { // Has the semaphore closed? - if curr & CLOSED > 0 { + if curr & Self::CLOSED > 0 { return Ready(Err(AcquireError::closed())); } let mut remaining = 0; let (next, acq) = if curr + acquired >= needed { let next = curr - (needed - acquired); - (next, needed >> 1) + (next, needed >> Self::PERMIT_SHIFT) } else { remaining = (needed - acquired) - curr; - (0, curr >> 1) + (0, curr >> Self::PERMIT_SHIFT) }; if remaining > 0 && lock.is_none() { From 4d84ec735a6583cf4cba9c5edb71dcb5c2b7f53d Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 23 Mar 2020 11:19:30 -0700 Subject: [PATCH 87/91] check before reregistering waker/reduce unsafe scope Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index c1b6b119236..b7f0de4d9c4 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -163,7 +163,7 @@ impl Semaphore { let mut curr = self.permits.load(Acquire); let num_permits = (num_permits as usize) << Self::PERMIT_SHIFT; loop { - // Has the semaphore closed? + // Has the semaphore closed?git if curr & Self::CLOSED > 0 { return Err(TryAcquireError::Closed); } @@ -294,19 +294,27 @@ impl Semaphore { assert_eq!(acquired, 0); // Otherwise, register the waker & enqueue the node. - unsafe { - node.waker.with_mut(|waker| { - // Safety: the wait list is locked, so we may modify the waker. + node.waker.with_mut(|waker| { + // Safety: the wait list is locked, so we may modify the waker. + let waker = unsafe { &mut *waker }; + // Do we need to register the new waker? + if waker + .as_ref() + .map(|waker| !waker.will_wake(cx.waker())) + .unwrap_or(true) + { *waker = Some(cx.waker().clone()); - }); + } + }); - // If the waiter is not already in the wait queue, enqueue it. - if !queued { + // If the waiter is not already in the wait queue, enqueue it. + if !queued { + let node = unsafe { let node = Pin::into_inner_unchecked(node) as *mut _; - let node = NonNull::new_unchecked(node); + NonNull::new_unchecked(node) + }; - waiters.queue.push_front(node); - } + waiters.queue.push_front(node); } Pending From f1e3c4c085badaa63a9f2f4fbdaeeea18f399468 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 23 Mar 2020 11:27:43 -0700 Subject: [PATCH 88/91] add overflow checks Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index b7f0de4d9c4..e0bcd44e2f8 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -107,7 +107,11 @@ impl Semaphore { /// Creates a new semaphore with the initial number of permits pub(crate) fn new(permits: usize) -> Self { - assert!(permits <= Self::MAX_PERMITS); + assert!( + permits <= Self::MAX_PERMITS, + "a semaphore may not have more than MAX_PERMITS permits ({})", + Self::MAX_PERMITS + ); Self { permits: AtomicUsize::new(permits << Self::PERMIT_SHIFT), waiters: Mutex::new(Waitlist { @@ -205,7 +209,19 @@ impl Semaphore { // If we assigned permits to all the waiters in the queue, and there are // still permits left over, assign them back to the semaphore. if rem > 0 { - self.permits.fetch_add(rem << Self::PERMIT_SHIFT, Release); + let permits = rem << Self::PERMIT_SHIFT; + assert!( + permits > Self::MAX_PERMITS, + "cannot add more than MAX_PERMITS permits ({})", + Self::MAX_PERMITS + ); + let prev = self.permits.fetch_add(rem << Self::PERMIT_SHIFT, Release); + assert!( + prev + permits <= Self::MAX_PERMITS, + "number of added permits ({}) would overflow MAX_PERMITS ({})", + rem, + Self::MAX_PERMITS + ); } // Split off the queue at the last waiter that was satisfied, creating a @@ -247,7 +263,10 @@ impl Semaphore { } let mut remaining = 0; - let (next, acq) = if curr + acquired >= needed { + let total = curr + .checked_add(acquired) + .expect("number of permits must not overflow"); + let (next, acq) = if total >= needed { let next = curr - (needed - acquired); (next, needed >> Self::PERMIT_SHIFT) } else { From a82c043f85e5d0a3fd67276e887be29466173254 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 23 Mar 2020 11:33:35 -0700 Subject: [PATCH 89/91] add comments explaining notification Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index e0bcd44e2f8..b6d10dbd52b 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -83,16 +83,6 @@ struct Waiter { _p: PhantomPinned, } -fn notify_all(mut list: LinkedList) { - while let Some(waiter) = list.pop_back() { - let waker = unsafe { waiter.as_ref().waker.with_mut(|waker| (*waker).take()) }; - - waker - .expect("if a node is in the wait list, it must have a waker") - .wake(); - } -} - impl Semaphore { /// The maximum number of permits which a semaphore can hold. /// @@ -190,6 +180,15 @@ impl Semaphore { Acquire::new(self, num_permits) } + /// Release `rem` permits to the semaphore's wait list, starting from the + /// end of the queue. + /// + /// This returns a new `LinkedList` containing all the waiters that received + /// enough permits to be notified. Once the lock on the wait list is + /// released, this list should be drained and the waiters in it notified. + /// + /// If `rem` exceeds the number of permits needed by the wait list, the + /// remainder are assigned back to the semaphore. fn add_permits_locked( &self, mut rem: usize, @@ -348,6 +347,18 @@ impl fmt::Debug for Semaphore { } } +/// Pop all waiters from `list`, starting at the end of the queue, and notify +/// them. +fn notify_all(mut list: LinkedList) { + while let Some(waiter) = list.pop_back() { + let waker = unsafe { waiter.as_ref().waker.with_mut(|waker| (*waker).take()) }; + + waker + .expect("if a node is in the wait list, it must have a waker") + .wake(); + } +} + impl Waiter { fn new(num_permits: u16) -> Self { Waiter { From 88fa1aa58e0e0693f9187179bf607bab00510978 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 23 Mar 2020 11:39:02 -0700 Subject: [PATCH 90/91] fix backwards assertion Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index b6d10dbd52b..a53e778cba6 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -210,7 +210,7 @@ impl Semaphore { if rem > 0 { let permits = rem << Self::PERMIT_SHIFT; assert!( - permits > Self::MAX_PERMITS, + permits < Self::MAX_PERMITS, "cannot add more than MAX_PERMITS permits ({})", Self::MAX_PERMITS ); From 449f8b10cc1f699c841b1b562e29aeeedba3df7d Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 23 Mar 2020 12:37:21 -0700 Subject: [PATCH 91/91] document future causalcell improvement Signed-off-by: Eliza Weisman --- tokio/src/sync/batch_semaphore.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index a53e778cba6..d89ac6ab885 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -69,7 +69,6 @@ struct Waiter { /// # Safety /// /// This may only be accessed while the wait queue is locked. - /// XXX: it would be nice if we could enforce this... waker: CausalCell>, /// Intrusive linked-list pointers. @@ -77,6 +76,13 @@ struct Waiter { /// # Safety /// /// This may only be accessed while the wait queue is locked. + /// + /// TODO: Ideally, we would be able to use loom to enforce that + /// this isn't accessed concurrently. However, it is difficult to + /// use a `CausalCell` here, since the `Link` trait requires _returning_ + /// references to `Pointers`, and `CausalCell` requires that checked access + /// take place inside a closure. We should consider changing `Pointers` to + /// use `CausalCell` internally. pointers: linked_list::Pointers, /// Should not be `Unpin`.