Futex Rework

src/backend/libc/thread/futex.rs

@@ -3,7 +3,7 @@ use crate::backend::c;
 bitflags::bitflags! {
     /// `FUTEX_*` flags for use with [`futex`].
     ///
-    /// [`futex`]: crate::thread::futex
+    /// [`futex`]: mod@crate::thread::futex
     #[repr(transparent)]
     #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
     pub struct FutexFlags: u32 {
@@ -16,7 +16,7 @@ bitflags::bitflags! {
 
 /// `FUTEX_*` operations for use with [`futex`].
 ///
-/// [`futex`]: crate::thread::futex
+/// [`futex`]: mod@crate::thread::futex
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
 #[repr(u32)]
 pub enum FutexOperation {
@@ -40,4 +40,18 @@ pub enum FutexOperation {
     TrylockPi = bitcast!(c::FUTEX_TRYLOCK_PI),
     /// `FUTEX_WAIT_BITSET`
     WaitBitset = bitcast!(c::FUTEX_WAIT_BITSET),
+    /// `FUTEX_WAKE_BITSET`
+    WakeBitset = bitcast!(c::FUTEX_WAKE_BITSET),
+    /// `FUTEX_WAIT_REQUEUE_PI`
+    WaitRequeuePi = bitcast!(c::FUTEX_WAIT_REQUEUE_PI),
+    /// `FUTEX_CMP_REQUEUE_PI`
+    CmpRequeuePi = bitcast!(c::FUTEX_CMP_REQUEUE_PI),
+    /// `FUTEX_LOCK_PI2`
+    LockPi2 = bitcast!(c::FUTEX_LOCK_PI2),
 }
+
+/// `FUTEX_WAITERS`
+pub const FUTEX_WAITERS: u32 = linux_raw_sys::general::FUTEX_WAITERS;
+
+/// `FUTEX_OWNER_DIED`
+pub const FUTEX_OWNER_DIED: u32 = linux_raw_sys::general::FUTEX_OWNER_DIED;

src/backend/libc/thread/syscalls.rs

@@ -8,12 +8,14 @@ use crate::thread::{NanosleepRelativeResult, Timespec};
 #[cfg(all(target_env = "gnu", fix_y2038))]
 use crate::timespec::LibcTimespec;
 use core::mem::MaybeUninit;
+use core::sync::atomic::AtomicU32;
 #[cfg(linux_kernel)]
 use {
+    super::futex::FutexOperation,
     crate::backend::conv::{borrowed_fd, ret_c_int, ret_usize},
     crate::fd::BorrowedFd,
     crate::pid::Pid,
-    crate::thread::{FutexFlags, FutexOperation},
+    crate::thread::FutexFlags,
     crate::utils::as_mut_ptr,
 };
 #[cfg(not(any(
@@ -415,15 +417,87 @@ pub(crate) fn setresgid_thread(
     unsafe { ret(setresgid(rgid.as_raw(), egid.as_raw(), sgid.as_raw())) }
 }
 
-// TODO: This could be de-multiplexed.
 #[cfg(linux_kernel)]
-pub(crate) unsafe fn futex(
-    uaddr: *mut u32,
+pub(crate) unsafe fn futex_val2(
+    uaddr: *const AtomicU32,
     op: FutexOperation,
     flags: FutexFlags,
     val: u32,
-    utime: *const Timespec,
-    uaddr2: *mut u32,
+    val2: u32,
+    uaddr2: *const AtomicU32,
+    val3: u32,
+) -> io::Result<usize> {
+    // the least significant four bytes of the timeout pointer are used as `val2`.
+    // ["the kernel casts the timeout value first to unsigned long, then to uint32_t"](https://man7.org/linux/man-pages/man2/futex.2.html),
+    // so we perform that exact conversion in reverse to create the pointer.
+    let timeout = val2 as usize as *const Timespec;
+
+    #[cfg(all(
+        target_pointer_width = "32",
+        not(any(target_arch = "aarch64", target_arch = "x86_64"))
+    ))]
+    {
+        // TODO: Upstream this to the libc crate.
+        #[allow(non_upper_case_globals)]
+        const SYS_futex_time64: i32 = linux_raw_sys::general::__NR_futex_time64 as i32;
+
+        syscall! {
+            fn futex_time64(
+                uaddr: *const AtomicU32,
+                futex_op: c::c_int,
+                val: u32,
+                timeout: *const Timespec,
+                uaddr2: *const AtomicU32,
+                val3: u32
+            ) via SYS_futex_time64 -> c::ssize_t
+        }
+
+        ret_usize(futex_time64(
+            uaddr,
+            op as i32 | flags.bits() as i32,
+            val,
+            timeout,
+            uaddr2,
+            val3,
+        ))
+    }
+
+    #[cfg(any(
+        target_pointer_width = "64",
+        target_arch = "aarch64",
+        target_arch = "x86_64"
+    ))]
+    {
+        syscall! {
+            fn futex(
+                uaddr: *const AtomicU32,
+                futex_op: c::c_int,
+                val: u32,
+                timeout: *const linux_raw_sys::general::__kernel_timespec,
+                uaddr2: *const AtomicU32,
+                val3: u32
+            ) via SYS_futex -> c::c_long
+        }
+
+        ret_usize(futex(
+            uaddr,
+            op as i32 | flags.bits() as i32,
+            val,
+            timeout.cast(),
+            uaddr2,
+            val3,
+        ) as isize)
+    }
+}
+
+#[cfg(linux_kernel)]
+pub(crate) unsafe fn futex_timeout(
+    uaddr: *const AtomicU32,
+    op: FutexOperation,
+    flags: FutexFlags,
+    val: u32,
+    timeout: *const Timespec,
+    uaddr2: *const AtomicU32,
     val3: u32,
 ) -> io::Result<usize> {
     #[cfg(all(
@@ -437,11 +511,11 @@ pub(crate) unsafe fn futex(
 
         syscall! {
             fn futex_time64(
-                uaddr: *mut u32,
+                uaddr: *const AtomicU32,
                 futex_op: c::c_int,
                 val: u32,
                 timeout: *const Timespec,
-                uaddr2: *mut u32,
+                uaddr2: *const AtomicU32,
                 val3: u32
             ) via SYS_futex_time64 -> c::ssize_t
         }
@@ -450,15 +524,15 @@ pub(crate) unsafe fn futex(
             uaddr,
             op as i32 | flags.bits() as i32,
             val,
-            utime,
+            timeout,
             uaddr2,
             val3,
         ))
         .or_else(|err| {
             // See the comments in `rustix_clock_gettime_via_syscall` about
             // emulation.
             if err == io::Errno::NOSYS {
-                futex_old(uaddr, op, flags, val, utime, uaddr2, val3)
+                futex_old_timespec(uaddr, op, flags, val, timeout, uaddr2, val3)
             } else {
                 Err(err)
             }
@@ -473,11 +547,11 @@ pub(crate) unsafe fn futex(
     {
         syscall! {
             fn futex(
-                uaddr: *mut u32,
+                uaddr: *const AtomicU32,
                 futex_op: c::c_int,
                 val: u32,
                 timeout: *const linux_raw_sys::general::__kernel_timespec,
-                uaddr2: *mut u32,
+                uaddr2: *const AtomicU32,
                 val3: u32
             ) via SYS_futex -> c::c_long
         }
@@ -486,7 +560,7 @@ pub(crate) unsafe fn futex(
             uaddr,
             op as i32 | flags.bits() as i32,
             val,
-            utime.cast(),
+            timeout.cast(),
             uaddr2,
             val3,
         ) as isize)
@@ -498,35 +572,45 @@ pub(crate) unsafe fn futex(
     target_pointer_width = "32",
     not(any(target_arch = "aarch64", target_arch = "x86_64"))
 ))]
-unsafe fn futex_old(
-    uaddr: *mut u32,
+unsafe fn futex_old_timespec(
+    uaddr: *const AtomicU32,
     op: FutexOperation,
     flags: FutexFlags,
     val: u32,
-    utime: *const Timespec,
-    uaddr2: *mut u32,
+    timeout: *const Timespec,
+    uaddr2: *const AtomicU32,
     val3: u32,
 ) -> io::Result<usize> {
     syscall! {
         fn futex(
-            uaddr: *mut u32,
+            uaddr: *const AtomicU32,
             futex_op: c::c_int,
             val: u32,
             timeout: *const linux_raw_sys::general::__kernel_old_timespec,
-            uaddr2: *mut u32,
+            uaddr2: *const AtomicU32,
             val3: u32
         ) via SYS_futex -> c::c_long
     }
 
-    let old_utime = linux_raw_sys::general::__kernel_old_timespec {
-        tv_sec: (*utime).tv_sec.try_into().map_err(|_| io::Errno::INVAL)?,
-        tv_nsec: (*utime).tv_nsec.try_into().map_err(|_| io::Errno::INVAL)?,
+    let old_timeout = if timeout.is_null() {
+        None
+    } else {
+        Some(linux_raw_sys::general::__kernel_old_timespec {
+            tv_sec: (*timeout).tv_sec.try_into().map_err(|_| io::Errno::INVAL)?,
+            tv_nsec: (*timeout)
+                .tv_nsec
+                .try_into()
+                .map_err(|_| io::Errno::INVAL)?,
+        })
     };
     ret_usize(futex(
         uaddr,
         op as i32 | flags.bits() as i32,
         val,
-        &old_utime,
+        old_timeout
+            .as_ref()
+            .map(|timeout| timeout as *const linux_raw_sys::general::__kernel_old_timespec)
+            .unwrap_or(core::ptr::null()),
         uaddr2,
         val3,
     ) as isize)

src/backend/linux_raw/conv.rs

@@ -790,11 +790,19 @@ impl<'a, Num: ArgNumber> From<(crate::net::SocketType, crate::net::SocketFlags)>
 }
 
 #[cfg(feature = "thread")]
-impl<'a, Num: ArgNumber> From<(crate::thread::FutexOperation, crate::thread::FutexFlags)>
-    for ArgReg<'a, Num>
+impl<'a, Num: ArgNumber>
+    From<(
+        crate::backend::thread::futex::FutexOperation,
+        crate::thread::FutexFlags,
+    )> for ArgReg<'a, Num>
 {
     #[inline]
-    fn from(pair: (crate::thread::FutexOperation, crate::thread::FutexFlags)) -> Self {
+    fn from(
+        pair: (
+            crate::backend::thread::futex::FutexOperation,
+            crate::thread::FutexFlags,
+        ),
+    ) -> Self {
         c_uint(pair.0 as u32 | pair.1.bits())
     }
 }

src/backend/linux_raw/thread/futex.rs

@@ -1,7 +1,7 @@
 bitflags::bitflags! {
     /// `FUTEX_*` flags for use with [`futex`].
     ///
-    /// [`futex`]: crate::thread::futex
+    /// [`futex`]: mod@crate::thread::futex
     #[repr(transparent)]
     #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
     pub struct FutexFlags: u32 {
@@ -18,7 +18,7 @@ bitflags::bitflags! {
 
 /// `FUTEX_*` operations for use with [`futex`].
 ///
-/// [`futex`]: crate::thread::futex
+/// [`futex`]: mod@crate::thread::futex
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
 #[repr(u32)]
 pub enum FutexOperation {
@@ -42,4 +42,18 @@ pub enum FutexOperation {
     TrylockPi = linux_raw_sys::general::FUTEX_TRYLOCK_PI,
     /// `FUTEX_WAIT_BITSET`
     WaitBitset = linux_raw_sys::general::FUTEX_WAIT_BITSET,
+    /// `FUTEX_WAKE_BITSET`
+    WakeBitset = linux_raw_sys::general::FUTEX_WAKE_BITSET,
+    /// `FUTEX_WAIT_REQUEUE_PI`
+    WaitRequeuePi = linux_raw_sys::general::FUTEX_WAIT_REQUEUE_PI,
+    /// `FUTEX_CMP_REQUEUE_PI`
+    CmpRequeuePi = linux_raw_sys::general::FUTEX_CMP_REQUEUE_PI,
+    /// `FUTEX_LOCK_PI2`
+    LockPi2 = linux_raw_sys::general::FUTEX_LOCK_PI2,
 }
+
+/// `FUTEX_WAITERS`
+pub const FUTEX_WAITERS: u32 = linux_raw_sys::general::FUTEX_WAITERS;
+
+/// `FUTEX_OWNER_DIED`
+pub const FUTEX_OWNER_DIED: u32 = linux_raw_sys::general::FUTEX_OWNER_DIED;