diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt index aa80d0e6..4b9abce6 100644 --- a/.github/.cspell/project-dictionary.txt +++ b/.github/.cspell/project-dictionary.txt @@ -61,6 +61,7 @@ metavar mfence mgba miscompilation +mmfr moreutils mstatus nand diff --git a/src/imp/atomic128/aarch64.rs b/src/imp/atomic128/aarch64.rs index 3e27dd49..68cd4120 100644 --- a/src/imp/atomic128/aarch64.rs +++ b/src/imp/atomic128/aarch64.rs @@ -46,7 +46,11 @@ include!("macros.rs"); #[cfg_attr(target_os = "windows", path = "detect/aarch64_windows.rs")] -#[cfg_attr(not(target_os = "windows"), path = "detect/aarch64_std.rs")] +#[cfg_attr(any(target_os = "freebsd", target_os = "openbsd"), path = "detect/aarch64_aa64reg.rs")] +#[cfg_attr( + not(any(target_os = "windows", target_os = "freebsd", target_os = "openbsd")), + path = "detect/aarch64_std.rs" +)] mod detect; #[cfg(not(portable_atomic_no_asm))] @@ -274,7 +278,14 @@ unsafe fn atomic_compare_exchange( #[cfg(not(all( not(portable_atomic_no_aarch64_target_feature), not(portable_atomic_no_outline_atomics), - any(feature = "std", target_os = "linux", target_os = "android", target_os = "windows", /* target_os = "freebsd" */) + any( + feature = "std", + target_os = "linux", + target_os = "android", + target_os = "windows", + target_os = "freebsd", + target_os = "openbsd", + ) )))] #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))] // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`. @@ -282,7 +293,14 @@ unsafe fn atomic_compare_exchange( #[cfg(all( not(portable_atomic_no_aarch64_target_feature), not(portable_atomic_no_outline_atomics), - any(feature = "std", target_os = "linux", target_os = "android", target_os = "windows", /* target_os = "freebsd" */) + any( + feature = "std", + target_os = "linux", + target_os = "android", + target_os = "windows", + target_os = "freebsd", + target_os = "openbsd", + ) ))] #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))] () => { diff --git a/src/imp/atomic128/detect/aarch64_aa64reg.rs b/src/imp/atomic128/detect/aarch64_aa64reg.rs new file mode 100644 index 00000000..c7194c40 --- /dev/null +++ b/src/imp/atomic128/detect/aarch64_aa64reg.rs @@ -0,0 +1,223 @@ +// Run-time feature detection on aarch64. +// +// As of nightly-2023-01-23, is_aarch64_feature_detected doesn't support run-time detection of FEAT_LSE on OpenBSD. +// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/mod.rs +// https://github.com/rust-lang/stdarch/pull/1374 +// +// Refs: +// - https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers?lang=en +// - https://www.kernel.org/doc/Documentation/arm64/cpu-feature-registers.txt +// - https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/aarch64.rs +// +// Supported platforms: +// - Linux 4.11+ +// https://github.com/torvalds/linux/commit/77c97b4ee21290f5f083173d957843b615abbff2 +// - FreeBSD 12.0+ +// https://github.com/freebsd/freebsd-src/commit/398810619cb32abf349f8de23f29510b2ee0839b +// - OpenBSD 7.1+ (through sysctl) +// https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8 +// +// For now, this module is only used on FreeBSD and OpenBSD, because on Linux +// this approach requires a higher kernel version than Rust supports, and also +// does not work with qemu-user as of QEMU 7.2. + +#![cfg_attr( + any( + portable_atomic_no_aarch64_target_feature, + portable_atomic_no_outline_atomics, + any(target_feature = "lse", portable_atomic_target_feature = "lse") + ), + allow(dead_code) +)] + +include!("common.rs"); + +struct AA64Reg { + aa64isar0: u64, + #[cfg(test)] + aa64mmfr2: u64, +} + +#[inline] +fn _detect(info: &mut CpuInfo) { + let AA64Reg { + aa64isar0, + #[cfg(test)] + aa64mmfr2, + } = imp::aa64reg(); + + // ID_AA64ISAR0_EL1, Instruction Set Attribute Register 0 + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/ID-AA64ISAR0-EL1--AArch64-Instruction-Set-Attribute-Register-0?lang=en + let atomic = extract(aa64isar0, 23, 20); + if atomic >= 2 { + info.set(CpuInfo::HAS_LSE); + #[cfg(test)] + { + if atomic >= 3 { + info.set(CpuInfo::HAS_LSE128); + } + } + } + + #[cfg(test)] + { + // ID_AA64MMFR2_EL1, AArch64 Memory Model Feature Register 2 + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/ID-AA64MMFR2-EL1--AArch64-Memory-Model-Feature-Register-2?lang=en + if extract(aa64mmfr2, 35, 32) >= 1 { + info.set(CpuInfo::HAS_LSE2); + } + } +} + +#[inline] +fn extract(x: u64, high: usize, low: usize) -> u64 { + (x >> low) & ((1 << (high - low + 1)) - 1) +} + +#[cfg(not(target_os = "openbsd"))] +mod imp { + #[cfg(not(portable_atomic_no_asm))] + use core::arch::asm; + + use super::AA64Reg; + + #[inline] + pub(super) fn aa64reg() -> AA64Reg { + // SAFETY: This is safe on FreeBSD 12.0+. FreeBSD 11 was EoL on 2021-09-30. + // Note that stdarch has been doing the same thing since before FreeBSD 11 was EoL. + // https://github.com/rust-lang/stdarch/pull/611 + unsafe { + let aa64isar0: u64; + asm!( + "mrs {}, ID_AA64ISAR0_EL1", + out(reg) aa64isar0, + options(pure, nomem, nostack, preserves_flags) + ); + #[cfg(test)] + let aa64mmfr2: u64; + #[cfg(test)] + { + asm!( + "mrs {}, ID_AA64MMFR2_EL1", + out(reg) aa64mmfr2, + options(pure, nomem, nostack, preserves_flags) + ); + } + AA64Reg { + aa64isar0, + #[cfg(test)] + aa64mmfr2, + } + } + } +} +// OpenBSD doesn't trap the mrs instruction, but exposes the system registers through sysctl. +// https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8 +// https://github.com/golang/go/commit/cd54ef1f61945459486e9eea2f016d99ef1da925 +#[cfg(target_os = "openbsd")] +mod imp { + use core::{mem::MaybeUninit, ptr}; + + use super::AA64Reg; + + // core::ffi::c_* (except c_void) requires Rust 1.64 + #[allow(non_camel_case_types)] + mod ffi { + pub(crate) use core::ffi::c_void; + // c_{,u}int is {i,u}32 on non-16-bit architectures + // https://github.com/rust-lang/rust/blob/1.67.0/library/core/src/ffi/mod.rs#L159-L173 + pub(crate) type c_int = i32; + pub(crate) type c_uint = u32; + // c_size_t is usize + // https://github.com/rust-lang/rust/blob/1.67.0/library/core/src/ffi/mod.rs#L83-L88 + pub(crate) type c_size_t = usize; + + extern "C" { + // https://man.openbsd.org/sysctl.2 + // https://github.com/rust-lang/libc/blob/0.2.139/src/unix/bsd/netbsdlike/openbsd/mod.rs#L1817-L1824 + pub(crate) fn sysctl( + name: *const c_int, + name_len: c_uint, + old_p: *mut c_void, + old_len_p: *mut c_size_t, + new_p: *mut c_void, + new_len: c_size_t, + ) -> c_int; + } + + // Defined in sys/sysctl.h. + // https://github.com/openbsd/src/blob/72ccc03bd11da614f31f7ff76e3f6fce99bc1c79/sys/sys/sysctl.h#L82 + pub(crate) const CTL_MACHDEP: c_int = 7; + // Defined in machine/cpu.h. + // https://github.com/openbsd/src/blob/72ccc03bd11da614f31f7ff76e3f6fce99bc1c79/sys/arch/arm64/include/cpu.h#L25-L40 + pub(crate) const CPU_ID_AA64ISAR0: c_int = 2; + #[cfg(test)] + pub(crate) const CPU_ID_AA64MMFR2: c_int = 7; + } + + // ID_AA64ISAR0_EL1 and ID_AA64ISAR1_EL1 are supported on OpenBSD 7.1+. + // https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8 + // Others are supported on OpenBSD 7.3+. + // https://github.com/openbsd/src/commit/c7654cd65262d532212f65123ee3905ba200365c + // sysctl returns an unsupported error if operation is not supported, + // so we can safely use this function on older versions of OpenBSD. + #[inline] + pub(super) fn aa64reg() -> AA64Reg { + let aa64isar0 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR0]).unwrap_or(0); + #[cfg(test)] + let aa64mmfr2 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64MMFR2]).unwrap_or(0); + AA64Reg { + aa64isar0, + #[cfg(test)] + aa64mmfr2, + } + } + + #[inline] + fn sysctl64(mib: &[ffi::c_int]) -> Option { + const OUT_LEN: ffi::c_size_t = core::mem::size_of::() as ffi::c_size_t; + let mut out = MaybeUninit::::uninit(); + let mut out_len = OUT_LEN; + #[allow(clippy::undocumented_unsafe_blocks)] // TODO + #[allow(clippy::cast_possible_truncation)] + let res = unsafe { + ffi::sysctl( + mib.as_ptr(), + mib.len() as ffi::c_uint, + out.as_mut_ptr() as *mut ffi::c_void, + &mut out_len, + ptr::null_mut(), + 0, + ) + }; + if res == -1 || out_len != OUT_LEN { + return None; + } + // SAFETY: we've checked that sysctl was successful and `out` was filled. + Some(unsafe { out.assume_init() }) + } +} + +#[allow(clippy::undocumented_unsafe_blocks)] +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_aa64reg() { + let AA64Reg { aa64isar0, aa64mmfr2 } = imp::aa64reg(); + std::eprintln!("aa64isar0={}", aa64isar0); + std::eprintln!("aa64mmfr2={}", aa64mmfr2); + if detect().test(CpuInfo::HAS_LSE) { + let atomic = extract(aa64isar0, 23, 20); + if detect().test(CpuInfo::HAS_LSE128) { + assert_eq!(atomic, 3); + } else { + assert_eq!(atomic, 2); + } + } + if detect().test(CpuInfo::HAS_LSE2) { + assert_eq!(extract(aa64mmfr2, 35, 32), 1); + } + } +} diff --git a/src/imp/atomic128/detect/aarch64_std.rs b/src/imp/atomic128/detect/aarch64_std.rs index c8fda023..87547d49 100644 --- a/src/imp/atomic128/detect/aarch64_std.rs +++ b/src/imp/atomic128/detect/aarch64_std.rs @@ -2,7 +2,14 @@ any( portable_atomic_no_aarch64_target_feature, portable_atomic_no_outline_atomics, - not(any(feature = "std", target_os = "linux", target_os = "android", target_os = "windows", /* target_os = "freebsd" */)), + not(any( + feature = "std", + target_os = "linux", + target_os = "android", + target_os = "windows", + // target_os = "freebsd", + // target_os = "openbsd", + )), any(target_feature = "lse", portable_atomic_target_feature = "lse"), ), allow(dead_code) @@ -23,8 +30,15 @@ pub(crate) fn has_lse() -> bool { not(portable_atomic_no_outline_atomics), // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/mod.rs // It is fine to use std for targets that we know can be linked to std. - // Note: aarch64 freebsd is tier 3, so std may not be available. - any(feature = "std", target_os = "linux", target_os = "android", target_os = "windows", /* target_os = "freebsd" */) + // Note: std may not be available on tier 3 such as aarch64 FreeBSD/OpenBSD. + any( + feature = "std", + target_os = "linux", + target_os = "android", + target_os = "windows", + // target_os = "freebsd", + // target_os = "openbsd", + ) ))] { extern crate std; diff --git a/src/imp/atomic128/detect/aarch64_windows.rs b/src/imp/atomic128/detect/aarch64_windows.rs index 0908c8c0..263037aa 100644 --- a/src/imp/atomic128/detect/aarch64_windows.rs +++ b/src/imp/atomic128/detect/aarch64_windows.rs @@ -1,6 +1,6 @@ // Run-time feature detection on aarch64 Windows. // -// As of nightly-2023-01-23, is_aarch64_feature_detected doesn't support detecting FEAT_LSE on Windows. +// As of nightly-2023-01-23, is_aarch64_feature_detected doesn't support run-time detection of FEAT_LSE on Windows. // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/windows/aarch64.rs // https://github.com/rust-lang/stdarch/pull/1373 // @@ -37,49 +37,3 @@ fn _detect(info: &mut CpuInfo) { info.set(CpuInfo::HAS_LSE); } } - -#[allow(clippy::undocumented_unsafe_blocks)] -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_bit_flags() { - let mut x = CpuInfo(0); - assert!(!x.test(CpuInfo::INIT)); - assert!(!x.test(CpuInfo::HAS_LSE)); - x.set(CpuInfo::INIT); - assert!(x.test(CpuInfo::INIT)); - assert!(!x.test(CpuInfo::HAS_LSE)); - x.set(CpuInfo::HAS_LSE); - assert!(x.test(CpuInfo::INIT)); - assert!(x.test(CpuInfo::HAS_LSE)); - } - - #[test] - fn test_detect() { - if has_lse() { - assert!(detect().test(CpuInfo::HAS_LSE)); - #[cfg(any( - target_feature = "lse", - portable_atomic_target_feature = "lse", - not(portable_atomic_no_aarch64_target_feature), - ))] - unsafe { - use core::{cell::UnsafeCell, sync::atomic::Ordering}; - let v = UnsafeCell::new(0); - assert_eq!( - super::super::_compare_exchange_casp(v.get(), 0, 1, Ordering::SeqCst), - 0 - ); - assert_eq!(*v.get(), 1); - } - } else { - assert!(!detect().test(CpuInfo::HAS_LSE)); - #[cfg(not(portable_atomic_no_aarch64_target_feature))] - { - assert!(!std::arch::is_aarch64_feature_detected!("lse")); - } - } - } -} diff --git a/src/imp/atomic128/detect/common.rs b/src/imp/atomic128/detect/common.rs index 1bc1f50c..e4d6250b 100644 --- a/src/imp/atomic128/detect/common.rs +++ b/src/imp/atomic128/detect/common.rs @@ -40,7 +40,16 @@ pub(crate) fn detect() -> CpuInfo { #[cfg(target_arch = "aarch64")] impl CpuInfo { + /// Whether FEAT_LSE is available const HAS_LSE: u32 = 1; + /// Whether FEAT_LSE2 is available + // This is currently only used in tests. + #[cfg(test)] + const HAS_LSE2: u32 = 2; + /// Whether FEAT_LSE128 is available + // This is currently only used in tests. + #[cfg(test)] + const HAS_LSE128: u32 = 3; } #[cfg(target_arch = "aarch64")] #[inline] @@ -58,7 +67,9 @@ pub(crate) fn has_lse() -> bool { #[cfg(target_arch = "x86_64")] impl CpuInfo { + /// Whether CMPXCHG16B is available const HAS_CMPXCHG16B: u32 = 1; + /// Whether VMOVDQA is atomic const HAS_VMOVDQA_ATOMIC: u32 = 2; #[allow(clippy::unused_self)] @@ -97,3 +108,83 @@ pub(crate) fn has_cmpxchg16b() -> bool { detect().has_cmpxchg16b() } } + +#[cfg(target_arch = "aarch64")] +#[allow(clippy::undocumented_unsafe_blocks)] +#[cfg(test)] +mod tests_aarch64_common { + use super::*; + + #[test] + fn test_bit_flags() { + let mut x = CpuInfo(0); + assert!(!x.test(CpuInfo::INIT)); + assert!(!x.test(CpuInfo::HAS_LSE)); + assert!(!x.test(CpuInfo::HAS_LSE2)); + assert!(!x.test(CpuInfo::HAS_LSE128)); + x.set(CpuInfo::INIT); + assert!(x.test(CpuInfo::INIT)); + assert!(!x.test(CpuInfo::HAS_LSE)); + assert!(!x.test(CpuInfo::HAS_LSE2)); + assert!(!x.test(CpuInfo::HAS_LSE128)); + x.set(CpuInfo::HAS_LSE); + assert!(x.test(CpuInfo::INIT)); + assert!(x.test(CpuInfo::HAS_LSE)); + assert!(!x.test(CpuInfo::HAS_LSE2)); + assert!(!x.test(CpuInfo::HAS_LSE128)); + x.set(CpuInfo::HAS_LSE2); + assert!(x.test(CpuInfo::INIT)); + assert!(x.test(CpuInfo::HAS_LSE)); + assert!(x.test(CpuInfo::HAS_LSE2)); + assert!(!x.test(CpuInfo::HAS_LSE128)); + x.set(CpuInfo::HAS_LSE128); + assert!(x.test(CpuInfo::INIT)); + assert!(x.test(CpuInfo::HAS_LSE)); + assert!(x.test(CpuInfo::HAS_LSE2)); + assert!(x.test(CpuInfo::HAS_LSE128)); + } + + #[test] + fn test_detect() { + if has_lse() { + assert!(detect().test(CpuInfo::HAS_LSE)); + #[cfg(any( + target_feature = "lse", + portable_atomic_target_feature = "lse", + not(portable_atomic_no_aarch64_target_feature), + ))] + unsafe { + use core::{cell::UnsafeCell, sync::atomic::Ordering}; + let v = UnsafeCell::new(0); + assert_eq!( + super::super::_compare_exchange_casp(v.get(), 0, 1, Ordering::SeqCst), + 0 + ); + assert_eq!(*v.get(), 1); + } + } else { + assert!(!detect().test(CpuInfo::HAS_LSE)); + #[cfg(not(portable_atomic_no_aarch64_target_feature))] + { + assert!(!std::arch::is_aarch64_feature_detected!("lse")); + } + } + if detect().test(CpuInfo::HAS_LSE2) { + assert!(detect().test(CpuInfo::HAS_LSE)); + assert!(detect().test(CpuInfo::HAS_LSE2)); + } else { + assert!(!detect().test(CpuInfo::HAS_LSE2)); + #[cfg(not(portable_atomic_no_aarch64_target_feature))] + { + assert!(!std::arch::is_aarch64_feature_detected!("lse2")); + } + } + if detect().test(CpuInfo::HAS_LSE128) { + assert!(detect().test(CpuInfo::HAS_LSE)); + assert!(detect().test(CpuInfo::HAS_LSE2)); + assert!(detect().test(CpuInfo::HAS_LSE128)); + } else { + assert!(!detect().test(CpuInfo::HAS_LSE128)); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 649609b2..395b658a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -146,6 +146,7 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a ) ))] #![warn( + improper_ctypes, missing_debug_implementations, missing_docs, rust_2018_idioms, diff --git a/tools/build.sh b/tools/build.sh index 95e2a205..86a53eae 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -51,7 +51,8 @@ default_targets=( aarch64-pc-windows-msvc aarch64-unknown-freebsd aarch64-unknown-linux-gnu - # aarch64 always support lse + aarch64-unknown-openbsd + # aarch64 always support lse & lse2 aarch64-apple-darwin # aarch64 big endian aarch64_be-unknown-linux-gnu