diff --git a/src/backend/libc/thread/futex.rs b/src/backend/libc/thread/futex.rs index 44d96f0f6..5bce85735 100644 --- a/src/backend/libc/thread/futex.rs +++ b/src/backend/libc/thread/futex.rs @@ -3,7 +3,7 @@ use crate::backend::c; bitflags::bitflags! { /// `FUTEX_*` flags for use with [`futex`]. /// - /// [`futex`]: crate::thread::futex + /// [`futex`]: mod@crate::thread::futex #[repr(transparent)] #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] pub struct FutexFlags: u32 { @@ -16,7 +16,7 @@ bitflags::bitflags! { /// `FUTEX_*` operations for use with [`futex`]. /// -/// [`futex`]: crate::thread::futex +/// [`futex`]: mod@crate::thread::futex #[derive(Debug, Copy, Clone, Eq, PartialEq)] #[repr(u32)] pub enum FutexOperation { @@ -40,4 +40,18 @@ pub enum FutexOperation { TrylockPi = bitcast!(c::FUTEX_TRYLOCK_PI), /// `FUTEX_WAIT_BITSET` WaitBitset = bitcast!(c::FUTEX_WAIT_BITSET), + /// `FUTEX_WAKE_BITSET` + WakeBitset = bitcast!(c::FUTEX_WAKE_BITSET), + /// `FUTEX_WAIT_REQUEUE_PI` + WaitRequeuePi = bitcast!(c::FUTEX_WAIT_REQUEUE_PI), + /// `FUTEX_CMP_REQUEUE_PI` + CmpRequeuePi = bitcast!(c::FUTEX_CMP_REQUEUE_PI), + /// `FUTEX_LOCK_PI2` + LockPi2 = bitcast!(c::FUTEX_LOCK_PI2), } + +/// `FUTEX_WAITERS` +pub const FUTEX_WAITERS: u32 = linux_raw_sys::general::FUTEX_WAITERS; + +/// `FUTEX_OWNER_DIED` +pub const FUTEX_OWNER_DIED: u32 = linux_raw_sys::general::FUTEX_OWNER_DIED; diff --git a/src/backend/libc/thread/syscalls.rs b/src/backend/libc/thread/syscalls.rs index df2378690..fb5c71115 100644 --- a/src/backend/libc/thread/syscalls.rs +++ b/src/backend/libc/thread/syscalls.rs @@ -8,12 +8,14 @@ use crate::thread::{NanosleepRelativeResult, Timespec}; #[cfg(all(target_env = "gnu", fix_y2038))] use crate::timespec::LibcTimespec; use core::mem::MaybeUninit; +use core::sync::atomic::AtomicU32; #[cfg(linux_kernel)] use { + super::futex::FutexOperation, crate::backend::conv::{borrowed_fd, ret_c_int, ret_usize}, crate::fd::BorrowedFd, crate::pid::Pid, - crate::thread::{FutexFlags, FutexOperation}, + crate::thread::FutexFlags, crate::utils::as_mut_ptr, }; #[cfg(not(any( @@ -415,15 +417,87 @@ pub(crate) fn setresgid_thread( unsafe { ret(setresgid(rgid.as_raw(), egid.as_raw(), sgid.as_raw())) } } -// TODO: This could be de-multiplexed. #[cfg(linux_kernel)] -pub(crate) unsafe fn futex( - uaddr: *mut u32, +pub(crate) unsafe fn futex_val2( + uaddr: *const AtomicU32, op: FutexOperation, flags: FutexFlags, val: u32, - utime: *const Timespec, - uaddr2: *mut u32, + val2: u32, + uaddr2: *const AtomicU32, + val3: u32, +) -> io::Result { + // the least significant four bytes of the timeout pointer are used as `val2`. + // ["the kernel casts the timeout value first to unsigned long, then to uint32_t"](https://man7.org/linux/man-pages/man2/futex.2.html), + // so we perform that exact conversion in reverse to create the pointer. + let timeout = val2 as usize as *const Timespec; + + #[cfg(all( + target_pointer_width = "32", + not(any(target_arch = "aarch64", target_arch = "x86_64")) + ))] + { + // TODO: Upstream this to the libc crate. + #[allow(non_upper_case_globals)] + const SYS_futex_time64: i32 = linux_raw_sys::general::__NR_futex_time64 as i32; + + syscall! { + fn futex_time64( + uaddr: *const AtomicU32, + futex_op: c::c_int, + val: u32, + timeout: *const Timespec, + uaddr2: *const AtomicU32, + val3: u32 + ) via SYS_futex_time64 -> c::ssize_t + } + + ret_usize(futex_time64( + uaddr, + op as i32 | flags.bits() as i32, + val, + timeout, + uaddr2, + val3, + )) + } + + #[cfg(any( + target_pointer_width = "64", + target_arch = "aarch64", + target_arch = "x86_64" + ))] + { + syscall! { + fn futex( + uaddr: *const AtomicU32, + futex_op: c::c_int, + val: u32, + timeout: *const linux_raw_sys::general::__kernel_timespec, + uaddr2: *const AtomicU32, + val3: u32 + ) via SYS_futex -> c::c_long + } + + ret_usize(futex( + uaddr, + op as i32 | flags.bits() as i32, + val, + timeout.cast(), + uaddr2, + val3, + ) as isize) + } +} + +#[cfg(linux_kernel)] +pub(crate) unsafe fn futex_timeout( + uaddr: *const AtomicU32, + op: FutexOperation, + flags: FutexFlags, + val: u32, + timeout: *const Timespec, + uaddr2: *const AtomicU32, val3: u32, ) -> io::Result { #[cfg(all( @@ -437,11 +511,11 @@ pub(crate) unsafe fn futex( syscall! { fn futex_time64( - uaddr: *mut u32, + uaddr: *const AtomicU32, futex_op: c::c_int, val: u32, timeout: *const Timespec, - uaddr2: *mut u32, + uaddr2: *const AtomicU32, val3: u32 ) via SYS_futex_time64 -> c::ssize_t } @@ -450,7 +524,7 @@ pub(crate) unsafe fn futex( uaddr, op as i32 | flags.bits() as i32, val, - utime, + timeout, uaddr2, val3, )) @@ -458,7 +532,7 @@ pub(crate) unsafe fn futex( // See the comments in `rustix_clock_gettime_via_syscall` about // emulation. if err == io::Errno::NOSYS { - futex_old(uaddr, op, flags, val, utime, uaddr2, val3) + futex_old_timespec(uaddr, op, flags, val, timeout, uaddr2, val3) } else { Err(err) } @@ -473,11 +547,11 @@ pub(crate) unsafe fn futex( { syscall! { fn futex( - uaddr: *mut u32, + uaddr: *const AtomicU32, futex_op: c::c_int, val: u32, timeout: *const linux_raw_sys::general::__kernel_timespec, - uaddr2: *mut u32, + uaddr2: *const AtomicU32, val3: u32 ) via SYS_futex -> c::c_long } @@ -486,7 +560,7 @@ pub(crate) unsafe fn futex( uaddr, op as i32 | flags.bits() as i32, val, - utime.cast(), + timeout.cast(), uaddr2, val3, ) as isize) @@ -498,35 +572,45 @@ pub(crate) unsafe fn futex( target_pointer_width = "32", not(any(target_arch = "aarch64", target_arch = "x86_64")) ))] -unsafe fn futex_old( - uaddr: *mut u32, +unsafe fn futex_old_timespec( + uaddr: *const AtomicU32, op: FutexOperation, flags: FutexFlags, val: u32, - utime: *const Timespec, - uaddr2: *mut u32, + timeout: *const Timespec, + uaddr2: *const AtomicU32, val3: u32, ) -> io::Result { syscall! { fn futex( - uaddr: *mut u32, + uaddr: *const AtomicU32, futex_op: c::c_int, val: u32, timeout: *const linux_raw_sys::general::__kernel_old_timespec, - uaddr2: *mut u32, + uaddr2: *const AtomicU32, val3: u32 ) via SYS_futex -> c::c_long } - let old_utime = linux_raw_sys::general::__kernel_old_timespec { - tv_sec: (*utime).tv_sec.try_into().map_err(|_| io::Errno::INVAL)?, - tv_nsec: (*utime).tv_nsec.try_into().map_err(|_| io::Errno::INVAL)?, + let old_timeout = if timeout.is_null() { + None + } else { + Some(linux_raw_sys::general::__kernel_old_timespec { + tv_sec: (*timeout).tv_sec.try_into().map_err(|_| io::Errno::INVAL)?, + tv_nsec: (*timeout) + .tv_nsec + .try_into() + .map_err(|_| io::Errno::INVAL)?, + }) }; ret_usize(futex( uaddr, op as i32 | flags.bits() as i32, val, - &old_utime, + old_timeout + .as_ref() + .map(|timeout| timeout as *const linux_raw_sys::general::__kernel_old_timespec) + .unwrap_or(core::ptr::null()), uaddr2, val3, ) as isize) diff --git a/src/backend/linux_raw/conv.rs b/src/backend/linux_raw/conv.rs index ab85c130f..9de51de3b 100644 --- a/src/backend/linux_raw/conv.rs +++ b/src/backend/linux_raw/conv.rs @@ -790,11 +790,19 @@ impl<'a, Num: ArgNumber> From<(crate::net::SocketType, crate::net::SocketFlags)> } #[cfg(feature = "thread")] -impl<'a, Num: ArgNumber> From<(crate::thread::FutexOperation, crate::thread::FutexFlags)> - for ArgReg<'a, Num> +impl<'a, Num: ArgNumber> + From<( + crate::backend::thread::futex::FutexOperation, + crate::thread::FutexFlags, + )> for ArgReg<'a, Num> { #[inline] - fn from(pair: (crate::thread::FutexOperation, crate::thread::FutexFlags)) -> Self { + fn from( + pair: ( + crate::backend::thread::futex::FutexOperation, + crate::thread::FutexFlags, + ), + ) -> Self { c_uint(pair.0 as u32 | pair.1.bits()) } } diff --git a/src/backend/linux_raw/thread/futex.rs b/src/backend/linux_raw/thread/futex.rs index 263e98070..ed1a34a7e 100644 --- a/src/backend/linux_raw/thread/futex.rs +++ b/src/backend/linux_raw/thread/futex.rs @@ -1,7 +1,7 @@ bitflags::bitflags! { /// `FUTEX_*` flags for use with [`futex`]. /// - /// [`futex`]: crate::thread::futex + /// [`futex`]: mod@crate::thread::futex #[repr(transparent)] #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] pub struct FutexFlags: u32 { @@ -18,7 +18,7 @@ bitflags::bitflags! { /// `FUTEX_*` operations for use with [`futex`]. /// -/// [`futex`]: crate::thread::futex +/// [`futex`]: mod@crate::thread::futex #[derive(Debug, Copy, Clone, Eq, PartialEq)] #[repr(u32)] pub enum FutexOperation { @@ -42,4 +42,18 @@ pub enum FutexOperation { TrylockPi = linux_raw_sys::general::FUTEX_TRYLOCK_PI, /// `FUTEX_WAIT_BITSET` WaitBitset = linux_raw_sys::general::FUTEX_WAIT_BITSET, + /// `FUTEX_WAKE_BITSET` + WakeBitset = linux_raw_sys::general::FUTEX_WAKE_BITSET, + /// `FUTEX_WAIT_REQUEUE_PI` + WaitRequeuePi = linux_raw_sys::general::FUTEX_WAIT_REQUEUE_PI, + /// `FUTEX_CMP_REQUEUE_PI` + CmpRequeuePi = linux_raw_sys::general::FUTEX_CMP_REQUEUE_PI, + /// `FUTEX_LOCK_PI2` + LockPi2 = linux_raw_sys::general::FUTEX_LOCK_PI2, } + +/// `FUTEX_WAITERS` +pub const FUTEX_WAITERS: u32 = linux_raw_sys::general::FUTEX_WAITERS; + +/// `FUTEX_OWNER_DIED` +pub const FUTEX_OWNER_DIED: u32 = linux_raw_sys::general::FUTEX_OWNER_DIED; diff --git a/src/backend/linux_raw/thread/syscalls.rs b/src/backend/linux_raw/thread/syscalls.rs index f1a8f6e71..907bb6457 100644 --- a/src/backend/linux_raw/thread/syscalls.rs +++ b/src/backend/linux_raw/thread/syscalls.rs @@ -5,6 +5,7 @@ //! See the `rustix::backend` module documentation for details. #![allow(unsafe_code, clippy::undocumented_unsafe_blocks)] +use super::futex::FutexOperation; use crate::backend::c; use crate::backend::conv::{ by_mut, by_ref, c_int, c_uint, ret, ret_c_int, ret_c_int_infallible, ret_usize, slice, @@ -13,8 +14,9 @@ use crate::backend::conv::{ use crate::fd::BorrowedFd; use crate::io; use crate::pid::Pid; -use crate::thread::{ClockId, FutexFlags, FutexOperation, NanosleepRelativeResult, Timespec}; +use crate::thread::{ClockId, FutexFlags, NanosleepRelativeResult, Timespec}; use core::mem::MaybeUninit; +use core::sync::atomic::AtomicU32; #[cfg(target_pointer_width = "32")] use linux_raw_sys::general::timespec as __kernel_old_timespec; use linux_raw_sys::general::{__kernel_timespec, TIMER_ABSTIME}; @@ -203,15 +205,53 @@ pub(crate) fn gettid() -> Pid { } } -// TODO: This could be de-multiplexed. #[inline] -pub(crate) unsafe fn futex( - uaddr: *mut u32, +pub(crate) unsafe fn futex_val2( + uaddr: *const AtomicU32, op: FutexOperation, flags: FutexFlags, val: u32, - utime: *const Timespec, - uaddr2: *mut u32, + val2: u32, + uaddr2: *const AtomicU32, + val3: u32, +) -> io::Result { + // the least significant four bytes of the timeout pointer are used as `val2`. + // ["the kernel casts the timeout value first to unsigned long, then to uint32_t"](https://man7.org/linux/man-pages/man2/futex.2.html), + // so we perform that exact conversion in reverse to create the pointer. + let timeout = val2 as usize as *const Timespec; + + #[cfg(target_pointer_width = "32")] + { + ret_usize(syscall!( + __NR_futex_time64, + uaddr, + (op, flags), + c_uint(val), + timeout, + uaddr2, + c_uint(val3) + )) + } + #[cfg(target_pointer_width = "64")] + ret_usize(syscall!( + __NR_futex, + uaddr, + (op, flags), + c_uint(val), + timeout, + uaddr2, + c_uint(val3) + )) +} + +#[inline] +pub(crate) unsafe fn futex_timeout( + uaddr: *const AtomicU32, + op: FutexOperation, + flags: FutexFlags, + val: u32, + timeout: *const Timespec, + uaddr2: *const AtomicU32, val3: u32, ) -> io::Result { #[cfg(target_pointer_width = "32")] @@ -221,7 +261,7 @@ pub(crate) unsafe fn futex( uaddr, (op, flags), c_uint(val), - utime, + timeout, uaddr2, c_uint(val3) )) @@ -229,7 +269,7 @@ pub(crate) unsafe fn futex( // See the comments in `rustix_clock_gettime_via_syscall` about // emulation. if err == io::Errno::NOSYS { - futex_old(uaddr, op, flags, val, utime, uaddr2, val3) + futex_old_timespec(uaddr, op, flags, val, timeout, uaddr2, val3) } else { Err(err) } @@ -241,32 +281,42 @@ pub(crate) unsafe fn futex( uaddr, (op, flags), c_uint(val), - utime, + timeout, uaddr2, c_uint(val3) )) } #[cfg(target_pointer_width = "32")] -unsafe fn futex_old( - uaddr: *mut u32, +unsafe fn futex_old_timespec( + uaddr: *const AtomicU32, op: FutexOperation, flags: FutexFlags, val: u32, - utime: *const Timespec, - uaddr2: *mut u32, + timeout: *const Timespec, + uaddr2: *const AtomicU32, val3: u32, ) -> io::Result { - let old_utime = __kernel_old_timespec { - tv_sec: (*utime).tv_sec.try_into().map_err(|_| io::Errno::INVAL)?, - tv_nsec: (*utime).tv_nsec.try_into().map_err(|_| io::Errno::INVAL)?, + let old_timeout = if timeout.is_null() { + None + } else { + Some(__kernel_old_timespec { + tv_sec: (*timeout).tv_sec.try_into().map_err(|_| io::Errno::INVAL)?, + tv_nsec: (*timeout) + .tv_nsec + .try_into() + .map_err(|_| io::Errno::INVAL)?, + }) }; ret_usize(syscall!( __NR_futex, uaddr, (op, flags), c_uint(val), - by_ref(&old_utime), + old_timeout + .as_ref() + .map(|timeout| timeout as *const __kernel_old_timespec) + .unwrap_or(core::ptr::null()), uaddr2, c_uint(val3) )) diff --git a/src/thread/futex.rs b/src/thread/futex.rs index 47947c8b5..beecc431f 100644 --- a/src/thread/futex.rs +++ b/src/thread/futex.rs @@ -6,11 +6,25 @@ //! primitives. #![allow(unsafe_code)] +use core::num::NonZeroU32; +use core::ptr; +use core::sync::atomic::AtomicU32; + +use crate::backend::thread::syscalls::{futex_timeout, futex_val2}; +use crate::fd::{FromRawFd, OwnedFd}; use crate::thread::Timespec; use crate::{backend, io}; -pub use backend::thread::futex::{FutexFlags, FutexOperation}; +pub use backend::thread::futex::FutexFlags; +pub use backend::thread::futex::FutexOperation; + +/// `FUTEX_WAITERS` +pub const FUTEX_WAITERS: u32 = backend::thread::futex::FUTEX_WAITERS; +/// `FUTEX_OWNER_DIED` +pub const FUTEX_OWNER_DIED: u32 = backend::thread::futex::FUTEX_OWNER_DIED; +/// DEPRECATED: There are now individual functions available to perform futex operations with improved type safety. See the [futex module](`self`). +/// /// `futex(uaddr, op, val, utime, uaddr2, val3)` /// /// # References @@ -24,6 +38,10 @@ pub use backend::thread::futex::{FutexFlags, FutexOperation}; /// /// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html /// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[deprecated( + since = "0.38.35", + note = "There are now individual functions available to perform futex operations with improved type safety. See the futex module." +)] #[inline] pub unsafe fn futex( uaddr: *mut u32, @@ -34,5 +52,574 @@ pub unsafe fn futex( uaddr2: *mut u32, val3: u32, ) -> io::Result { - backend::thread::syscalls::futex(uaddr, op, flags, val, utime, uaddr2, val3) + use FutexOperation::*; + + match op { + Wait | LockPi | WaitBitset | WaitRequeuePi | LockPi2 => futex_timeout( + uaddr as *const AtomicU32, + op, + flags, + val, + utime, + uaddr2 as *const AtomicU32, + val3, + ), + Wake | Fd | Requeue | CmpRequeue | WakeOp | UnlockPi | TrylockPi | WakeBitset + | CmpRequeuePi => futex_val2( + uaddr as *const AtomicU32, + op, + flags, + val, + utime as usize as u32, + uaddr2 as *const AtomicU32, + val3, + ), + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAIT, val, timeout, NULL, 0)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wait( + uaddr: &AtomicU32, + flags: FutexFlags, + val: u32, + timeout: Option, +) -> io::Result<()> { + unsafe { + backend::thread::syscalls::futex_timeout( + uaddr, + FutexOperation::Wait, + flags, + val, + timeout + .as_ref() + .map(|timeout| timeout as *const Timespec) + .unwrap_or(ptr::null()), + ptr::null(), + 0, + ) + .map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAKE, val, NULL, NULL, 0)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wake(uaddr: &AtomicU32, flags: FutexFlags, val: u32) -> io::Result { + unsafe { + backend::thread::syscalls::futex_val2( + uaddr, + FutexOperation::Wake, + flags, + val, + 0, + ptr::null(), + 0, + ) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_FD, val, NULL, NULL, 0)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn fd(uaddr: &AtomicU32, flags: FutexFlags, val: u32) -> io::Result { + unsafe { + backend::thread::syscalls::futex_val2( + uaddr, + FutexOperation::Fd, + flags, + val, + 0, + ptr::null(), + 0, + ) + .map(|fd| OwnedFd::from_raw_fd(fd.try_into().expect("return value should be a valid fd"))) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_REQUEUE, val, val2, uaddr2, 0)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn requeue( + uaddr: &AtomicU32, + flags: FutexFlags, + val: u32, + val2: u32, + uaddr2: &AtomicU32, +) -> io::Result { + unsafe { + backend::thread::syscalls::futex_val2( + uaddr, + FutexOperation::Requeue, + flags, + val, + val2, + uaddr2, + 0, + ) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_CMP_REQUEUE, val, val2, uaddr2, val3)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn cmp_requeue( + uaddr: &AtomicU32, + flags: FutexFlags, + val: u32, + val2: u32, + uaddr2: &AtomicU32, + val3: u32, +) -> io::Result { + unsafe { + backend::thread::syscalls::futex_val2( + uaddr, + FutexOperation::CmpRequeue, + flags, + val, + val2, + uaddr2, + val3, + ) + } +} + +/// `FUTEX_OP_*` operations for use with [`wake_op`]. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[repr(u32)] +pub enum WakeOp { + /// `FUTEX_OP_SET`: `uaddr2 = oparg;` + Set = 0, + /// `FUTEX_OP_ADD`: `uaddr2 += oparg;` + Add = 1, + /// `FUTEX_OP_OR`: `uaddr2 |= oparg;` + Or = 2, + /// `FUTEX_OP_ANDN`: `uaddr2 &= ~oparg;` + AndN = 3, + /// `FUTEX_OP_XOR`: `uaddr2 ^= oparg;` + XOr = 4, + /// `FUTEX_OP_SET | FUTEX_OP_ARG_SHIFT`: `uaddr2 = (oparg << 1);` + SetShift = 0 | 8, + /// `FUTEX_OP_ADD | FUTEX_OP_ARG_SHIFT`: `uaddr2 += (oparg << 1);` + AddShift = 1 | 8, + /// `FUTEX_OP_OR | FUTEX_OP_ARG_SHIFT`: `uaddr2 |= (oparg << 1);` + OrShift = 2 | 8, + /// `FUTEX_OP_ANDN | FUTEX_OP_ARG_SHIFT`: `uaddr2 &= !(oparg << 1);` + AndNShift = 3 | 8, + /// `FUTEX_OP_XOR | FUTEX_OP_ARG_SHIFT`: `uaddr2 ^= (oparg << 1);` + XOrShift = 4 | 8, +} + +/// `FUTEX_OP_CMP_*` operations for use with [`wake_op`]. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[repr(u32)] +pub enum WakeOpCmp { + /// `FUTEX_OP_CMP_EQ`: `if oldval == cmparg { wake(); }` + Eq = 0, + /// `FUTEX_OP_CMP_EQ`: `if oldval != cmparg { wake(); }` + Ne = 1, + /// `FUTEX_OP_CMP_EQ`: `if oldval < cmparg { wake(); }` + Lt = 2, + /// `FUTEX_OP_CMP_EQ`: `if oldval <= cmparg { wake(); }` + Le = 3, + /// `FUTEX_OP_CMP_EQ`: `if oldval > cmparg { wake(); }` + Gt = 4, + /// `FUTEX_OP_CMP_EQ`: `if oldval >= cmparg { wake(); }` + Ge = 5, +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAKE_OP, val, val2, uaddr2, val3)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wake_op( + uaddr: &AtomicU32, + flags: FutexFlags, + val: u32, + val2: u32, + uaddr2: &AtomicU32, + op: WakeOp, + cmp: WakeOpCmp, + oparg: u16, + cmparg: u16, +) -> io::Result { + if oparg >= 1 << 12 || cmparg >= 1 << 12 { + return Err(io::Errno::INVAL); + } + + let val3 = + ((op as u32) << 28) | ((cmp as u32) << 24) | ((oparg as u32) << 12) | (cmparg as u32); + + unsafe { + backend::thread::syscalls::futex_val2( + uaddr, + FutexOperation::WakeOp, + flags, + val, + val2, + uaddr2, + val3, + ) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_LOCK_PI, 0, timeout, NULL, 0)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn lock_pi(uaddr: &AtomicU32, flags: FutexFlags, timeout: Option) -> io::Result<()> { + unsafe { + backend::thread::syscalls::futex_timeout( + uaddr, + FutexOperation::LockPi, + flags, + 0, + timeout + .as_ref() + .map(|timeout| timeout as *const Timespec) + .unwrap_or(ptr::null()), + ptr::null(), + 0, + ) + .map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_UNLOCK_PI, 0, NULL, NULL, 0)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn unlock_pi(uaddr: &AtomicU32, flags: FutexFlags) -> io::Result<()> { + unsafe { + backend::thread::syscalls::futex_val2( + uaddr, + FutexOperation::UnlockPi, + flags, + 0, + 0, + ptr::null(), + 0, + ) + .map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_TRYLOCK_PI, 0, NULL, NULL, 0)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn trylock_pi(uaddr: &AtomicU32, flags: FutexFlags) -> io::Result { + unsafe { + backend::thread::syscalls::futex_val2( + uaddr, + FutexOperation::TrylockPi, + flags, + 0, + 0, + ptr::null(), + 0, + ) + .map(|ret| ret == 0) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAIT_BITSET, val, timeout/val2, NULL, val3)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wait_bitset( + uaddr: &AtomicU32, + flags: FutexFlags, + val: u32, + timeout: Option, + val3: NonZeroU32, +) -> io::Result<()> { + unsafe { + backend::thread::syscalls::futex_timeout( + uaddr, + FutexOperation::WaitBitset, + flags, + val, + timeout + .as_ref() + .map(|timeout| timeout as *const Timespec) + .unwrap_or(ptr::null()), + ptr::null(), + val3.get(), + ) + .map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAKE_BITSET, val, NULL, NULL, val3)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wake_bitset( + uaddr: &AtomicU32, + flags: FutexFlags, + val: u32, + val3: NonZeroU32, +) -> io::Result { + unsafe { + backend::thread::syscalls::futex_val2( + uaddr, + FutexOperation::WakeBitset, + flags, + val, + 0, + ptr::null(), + val3.get(), + ) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAIT_REQUEUE_PI, val, timeout, uaddr2, 0)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wait_requeue_pi( + uaddr: &AtomicU32, + flags: FutexFlags, + val: u32, + timeout: Option, + uaddr2: &AtomicU32, +) -> io::Result<()> { + unsafe { + backend::thread::syscalls::futex_timeout( + uaddr, + FutexOperation::WaitRequeuePi, + flags, + val, + timeout + .as_ref() + .map(|timeout| timeout as *const Timespec) + .unwrap_or(ptr::null()), + uaddr2, + 0, + ) + .map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_CMP_REQUEUE_PI, 1, val2, uaddr2, val3)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn cmp_requeue_pi( + uaddr: &AtomicU32, + flags: FutexFlags, + val2: u32, + uaddr2: &AtomicU32, + val3: u32, +) -> io::Result { + unsafe { + backend::thread::syscalls::futex_val2( + uaddr, + FutexOperation::CmpRequeuePi, + flags, + 1, + val2, + uaddr2, + val3, + ) + } +} + +/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_LOCK_PI2, 0, timeout, NULL, 0)` +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// # Safety +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links above. +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn lock_pi2(uaddr: &AtomicU32, flags: FutexFlags, timeout: Option) -> io::Result<()> { + unsafe { + backend::thread::syscalls::futex_timeout( + uaddr, + FutexOperation::LockPi2, + flags, + 0, + timeout + .as_ref() + .map(|timeout| timeout as *const Timespec) + .unwrap_or(ptr::null()), + ptr::null(), + 0, + ) + .map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } } diff --git a/src/thread/mod.rs b/src/thread/mod.rs index f3b8b3ebd..cb5d566f3 100644 --- a/src/thread/mod.rs +++ b/src/thread/mod.rs @@ -3,7 +3,7 @@ #[cfg(not(target_os = "redox"))] mod clock; #[cfg(linux_kernel)] -mod futex; +pub mod futex; #[cfg(linux_kernel)] mod id; #[cfg(linux_kernel)] @@ -16,7 +16,8 @@ mod setns; #[cfg(not(target_os = "redox"))] pub use clock::*; #[cfg(linux_kernel)] -pub use futex::{futex, FutexFlags, FutexOperation}; +#[allow(deprecated)] +pub use futex::{futex, FutexFlags, FutexOperation, FUTEX_OWNER_DIED, FUTEX_WAITERS}; #[cfg(linux_kernel)] pub use id::{ gettid, set_thread_gid, set_thread_groups, set_thread_res_gid, set_thread_res_uid, diff --git a/tests/thread/futex.rs b/tests/thread/futex.rs new file mode 100644 index 000000000..ab5b937f9 --- /dev/null +++ b/tests/thread/futex.rs @@ -0,0 +1,108 @@ +use core::sync::atomic::{AtomicU32, Ordering}; +use rustix::{ + io::Errno, + thread::{futex, FutexFlags}, +}; + +#[test] +fn test_lock_unlock_pi() { + let lock = AtomicU32::new(0); + futex::lock_pi(&lock, FutexFlags::empty(), None).unwrap(); + assert_ne!(lock.load(Ordering::SeqCst), 0); + + let err = unsafe { futex::lock_pi(&lock, FutexFlags::empty(), None).unwrap_err() }; + assert_eq!(err, Errno::DEADLK); + + futex::unlock_pi(&lock, FutexFlags::empty()).unwrap(); + assert_eq!(lock.load(Ordering::SeqCst), 0); + + let err = futex::unlock_pi(&lock, FutexFlags::empty()).unwrap_err(); + assert_eq!(err, Errno::PERM); +} + +#[cfg(feature = "std")] +#[test] +fn test_wait_wake() { + let lock = std::sync::Arc::new(AtomicU32::new(0)); + + match futex::wait(&lock, FutexFlags::empty(), 1, None) { + Ok(()) => panic!("Nobody should be waking us!"), + Err(Errno::AGAIN) => { + assert_eq!(lock.load(Ordering::SeqCst), 0, "the lock should still be 0") + } + Err(err) => panic!("{err}"), + } + + let other = std::thread::spawn({ + let lock = lock.clone(); + move || { + std::thread::sleep(std::time::Duration::from_millis(1)); + lock.store(1, Ordering::SeqCst); + futex::wake(&lock, FutexFlags::empty(), 1); + + std::thread::sleep(std::time::Duration::from_millis(50)); + match futex::wait(&lock, FutexFlags::empty(), 1, None) { + Ok(()) => (), + Err(Errno::AGAIN) => { + assert_eq!(lock.load(Ordering::SeqCst), 2, "the lock should now be 2") + } + Err(err) => panic!("{err}"), + } + } + }); + + match futex::wait(&lock, FutexFlags::empty(), 0, None) { + Ok(()) => (), + Err(Errno::AGAIN) => assert_eq!(lock.load(Ordering::SeqCst), 1, "the lock should now be 1"), + Err(err) => panic!("{err}"), + } + + lock.store(2, Ordering::SeqCst); + futex::wake(&lock, FutexFlags::empty(), 1); + + other.join().unwrap(); +} + +#[cfg(feature = "std")] +#[test] +fn test_timeout() { + use rustix::fs::Timespec; + + let lock = AtomicU32::new(0); + + let err = futex::wait( + &lock, + FutexFlags::empty(), + 0, + Some(Timespec { + tv_sec: 1, + tv_nsec: 13, + }), + ) + .unwrap_err(); + assert_eq!(err, Errno::TIMEDOUT); + + let err = futex::wait( + &lock, + FutexFlags::empty(), + 0, + Some(Timespec { + tv_sec: 0, + tv_nsec: 1_000_000_000, + }), + ) + .unwrap_err(); + assert_eq!(err, Errno::INVAL); + + let err = futex::wait( + &lock, + FutexFlags::empty(), + 0, + Some(Timespec { + tv_sec: -1, + tv_nsec: 0, + }), + ) + .unwrap_err(); + assert_eq!(err, Errno::INVAL); +} diff --git a/tests/thread/main.rs b/tests/thread/main.rs index fccf2682c..bdd02dc6c 100644 --- a/tests/thread/main.rs +++ b/tests/thread/main.rs @@ -6,6 +6,8 @@ #[cfg(not(target_os = "redox"))] mod clocks; #[cfg(linux_kernel)] +mod futex; +#[cfg(linux_kernel)] mod id; #[cfg(linux_kernel)] mod libcap;