diff --git a/src/imp/atomic128/detect/aarch64_windows.rs b/src/imp/atomic128/detect/aarch64_windows.rs index fee7212a..0908c8c0 100644 --- a/src/imp/atomic128/detect/aarch64_windows.rs +++ b/src/imp/atomic128/detect/aarch64_windows.rs @@ -17,26 +17,6 @@ include!("common.rs"); -use core::sync::atomic::{AtomicU32, Ordering}; - -impl CpuInfo { - const INIT: u32 = 0; - const HAS_LSE: u32 = 1; -} - -#[inline] -pub(crate) fn has_lse() -> bool { - #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))] - { - // FEAT_LSE is statically available. - true - } - #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))] - { - detect().test(CpuInfo::HAS_LSE) - } -} - #[allow(clippy::upper_case_acronyms)] #[inline] fn _detect(info: &mut CpuInfo) { @@ -51,7 +31,6 @@ fn _detect(info: &mut CpuInfo) { fn IsProcessorFeaturePresent(ProcessorFeature: DWORD) -> BOOL; } - info.set(CpuInfo::INIT); // SAFETY: calling IsProcessorFeaturePresent is safe, and FALSE is also // returned if the HAL does not support detection of the specified feature. if unsafe { IsProcessorFeaturePresent(PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE) != FALSE } { @@ -59,18 +38,6 @@ fn _detect(info: &mut CpuInfo) { } } -#[inline] -fn detect() -> CpuInfo { - static CACHE: AtomicU32 = AtomicU32::new(0); - let mut info = CpuInfo(CACHE.load(Ordering::Relaxed)); - if info.0 != 0 { - return info; - } - _detect(&mut info); - CACHE.store(info.0, Ordering::Relaxed); - info -} - #[allow(clippy::undocumented_unsafe_blocks)] #[cfg(test)] mod tests { @@ -99,7 +66,7 @@ mod tests { not(portable_atomic_no_aarch64_target_feature), ))] unsafe { - use core::cell::UnsafeCell; + use core::{cell::UnsafeCell, sync::atomic::Ordering}; let v = UnsafeCell::new(0); assert_eq!( super::super::_compare_exchange_casp(v.get(), 0, 1, Ordering::SeqCst), diff --git a/src/imp/atomic128/detect/common.rs b/src/imp/atomic128/detect/common.rs index 366bca86..1bc1f50c 100644 --- a/src/imp/atomic128/detect/common.rs +++ b/src/imp/atomic128/detect/common.rs @@ -2,6 +2,8 @@ pub(crate) struct CpuInfo(u32); impl CpuInfo { + const INIT: u32 = 0; + #[inline] fn set(&mut self, bit: u32) { self.0 = set(self.0, bit); @@ -20,3 +22,78 @@ fn set(x: u32, bit: u32) -> u32 { fn test(x: u32, bit: u32) -> bool { x & (1 << bit) != 0 } + +#[inline] +pub(crate) fn detect() -> CpuInfo { + use core::sync::atomic::{AtomicU32, Ordering}; + + static CACHE: AtomicU32 = AtomicU32::new(0); + let mut info = CpuInfo(CACHE.load(Ordering::Relaxed)); + if info.0 != 0 { + return info; + } + info.set(CpuInfo::INIT); + _detect(&mut info); + CACHE.store(info.0, Ordering::Relaxed); + info +} + +#[cfg(target_arch = "aarch64")] +impl CpuInfo { + const HAS_LSE: u32 = 1; +} +#[cfg(target_arch = "aarch64")] +#[inline] +pub(crate) fn has_lse() -> bool { + #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))] + { + // FEAT_LSE is statically available. + true + } + #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))] + { + detect().test(CpuInfo::HAS_LSE) + } +} + +#[cfg(target_arch = "x86_64")] +impl CpuInfo { + const HAS_CMPXCHG16B: u32 = 1; + const HAS_VMOVDQA_ATOMIC: u32 = 2; + + #[allow(clippy::unused_self)] + #[inline] + pub(crate) fn has_cmpxchg16b(self) -> bool { + #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] + { + // CMPXCHG16B is statically available. + true + } + #[cfg(not(any( + target_feature = "cmpxchg16b", + portable_atomic_target_feature = "cmpxchg16b" + )))] + { + self.test(CpuInfo::HAS_CMPXCHG16B) + } + } + #[inline] + pub(crate) fn has_vmovdqa_atomic(self) -> bool { + self.test(CpuInfo::HAS_VMOVDQA_ATOMIC) + } +} +/// Equivalent to `detect().has_cmpxchg16b()`, but avoids calling `detect()` +/// if CMPXCHG16B is statically available. +#[cfg(target_arch = "x86_64")] +#[inline] +pub(crate) fn has_cmpxchg16b() -> bool { + #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] + { + // CMPXCHG16B is statically available. + true + } + #[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))] + { + detect().has_cmpxchg16b() + } +} diff --git a/src/imp/atomic128/detect/x86_64.rs b/src/imp/atomic128/detect/x86_64.rs index 4acdcd0e..df083b34 100644 --- a/src/imp/atomic128/detect/x86_64.rs +++ b/src/imp/atomic128/detect/x86_64.rs @@ -14,37 +14,7 @@ include!("common.rs"); #[cfg(not(portable_atomic_no_asm))] use core::arch::asm; -use core::{ - arch::x86_64::CpuidResult, - sync::atomic::{AtomicU32, Ordering}, -}; - -impl CpuInfo { - const INIT: u32 = 0; - const HAS_CMPXCHG16B: u32 = 1; - const HAS_VMOVDQA_ATOMIC: u32 = 2; - - #[allow(clippy::unused_self)] - #[inline] - pub(crate) fn has_cmpxchg16b(self) -> bool { - #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] - { - // CMPXCHG16B is statically available. - true - } - #[cfg(not(any( - target_feature = "cmpxchg16b", - portable_atomic_target_feature = "cmpxchg16b" - )))] - { - self.test(CpuInfo::HAS_CMPXCHG16B) - } - } - #[inline] - pub(crate) fn has_vmovdqa_atomic(self) -> bool { - self.test(CpuInfo::HAS_VMOVDQA_ATOMIC) - } -} +use core::arch::x86_64::CpuidResult; // Workaround for https://github.com/rust-lang/rust/issues/101346 // It is not clear if our use cases are affected, but we implement this just in case. @@ -93,8 +63,7 @@ unsafe fn _vendor_id() -> [u8; 12] { } #[inline] -fn _cpuid(info: &mut CpuInfo) { - info.set(CpuInfo::INIT); +fn _detect(info: &mut CpuInfo) { // Miri doesn't support inline assembly used in __cpuid #[cfg(miri)] { @@ -136,33 +105,6 @@ fn _cpuid(info: &mut CpuInfo) { } } -#[inline] -pub(crate) fn cpuid() -> CpuInfo { - static CACHE: AtomicU32 = AtomicU32::new(0); - let mut info = CpuInfo(CACHE.load(Ordering::Relaxed)); - if info.0 != 0 { - return info; - } - _cpuid(&mut info); - CACHE.store(info.0, Ordering::Relaxed); - info -} - -/// Equivalent to `cpuid().has_cmpxchg16b()`, but avoids calling `cpuid()` -/// if CMPXCHG16B is statically available. -#[inline] -pub(crate) fn has_cmpxchg16b() -> bool { - #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] - { - // CMPXCHG16B is statically available. - true - } - #[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))] - { - cpuid().has_cmpxchg16b() - } -} - #[allow(clippy::undocumented_unsafe_blocks)] #[cfg(test)] mod tests { @@ -196,9 +138,9 @@ mod tests { assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), has_cmpxchg16b()); let vendor_id = unsafe { _vendor_id() }; if vendor_id == VENDOR_ID_INTEL || vendor_id == VENDOR_ID_AMD { - assert_eq!(std::is_x86_feature_detected!("avx"), cpuid().has_vmovdqa_atomic()); + assert_eq!(std::is_x86_feature_detected!("avx"), detect().has_vmovdqa_atomic()); } else { - assert!(!cpuid().has_vmovdqa_atomic()); + assert!(!detect().has_vmovdqa_atomic()); } } } diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs index 215ec5f3..52e58a96 100644 --- a/src/imp/atomic128/x86_64.rs +++ b/src/imp/atomic128/x86_64.rs @@ -312,7 +312,7 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 { ifunc!(unsafe fn(src: *mut u128, order: Ordering) -> u128; { // Check CMPXCHG16B anyway to prevent mixing atomic and non-atomic access. - let cpuid = detect::cpuid(); + let cpuid = detect::detect(); if cpuid.has_cmpxchg16b() && cpuid.has_vmovdqa_atomic() { _atomic_load_vmovdqa } else { @@ -357,7 +357,7 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) { ifunc!(unsafe fn(dst: *mut u128, val: u128, order: Ordering); { // Check CMPXCHG16B anyway to prevent mixing atomic and non-atomic access. - let cpuid = detect::cpuid(); + let cpuid = detect::detect(); if cpuid.has_cmpxchg16b() && cpuid.has_vmovdqa_atomic() { _atomic_store_vmovdqa } else {