Skip to content

Commit

Permalink
detect: Move more code to common.rs
Browse files Browse the repository at this point in the history
  • Loading branch information
@taiki-e
taiki-e committed Jan 24, 2023
1 parent e5463e2 commit de86109
Show file tree
Hide file tree
Showing 4 changed files with 84 additions and 98 deletions.
35 changes: 1 addition & 34 deletions src/imp/atomic128/detect/aarch64_windows.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,6 @@

include!("common.rs");

use core::sync::atomic::{AtomicU32, Ordering};

impl CpuInfo {
const INIT: u32 = 0;
const HAS_LSE: u32 = 1;
}

#[inline]
pub(crate) fn has_lse() -> bool {
#[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
{
// FEAT_LSE is statically available.
true
}
#[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
{
detect().test(CpuInfo::HAS_LSE)
}
}

#[allow(clippy::upper_case_acronyms)]
#[inline]
fn _detect(info: &mut CpuInfo) {
Expand All @@ -51,26 +31,13 @@ fn _detect(info: &mut CpuInfo) {
fn IsProcessorFeaturePresent(ProcessorFeature: DWORD) -> BOOL;
}

info.set(CpuInfo::INIT);
// SAFETY: calling IsProcessorFeaturePresent is safe, and FALSE is also
// returned if the HAL does not support detection of the specified feature.
if unsafe { IsProcessorFeaturePresent(PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE) != FALSE } {
info.set(CpuInfo::HAS_LSE);
}
}

#[inline]
fn detect() -> CpuInfo {
static CACHE: AtomicU32 = AtomicU32::new(0);
let mut info = CpuInfo(CACHE.load(Ordering::Relaxed));
if info.0 != 0 {
return info;
}
_detect(&mut info);
CACHE.store(info.0, Ordering::Relaxed);
info
}

#[allow(clippy::undocumented_unsafe_blocks)]
#[cfg(test)]
mod tests {
Expand Down Expand Up @@ -99,7 +66,7 @@ mod tests {
not(portable_atomic_no_aarch64_target_feature),
))]
unsafe {
use core::cell::UnsafeCell;
use core::{cell::UnsafeCell, sync::atomic::Ordering};
let v = UnsafeCell::new(0);
assert_eq!(
super::super::_compare_exchange_casp(v.get(), 0, 1, Ordering::SeqCst),
Expand Down
77 changes: 77 additions & 0 deletions src/imp/atomic128/detect/common.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
pub(crate) struct CpuInfo(u32);

impl CpuInfo {
const INIT: u32 = 0;

#[inline]
fn set(&mut self, bit: u32) {
self.0 = set(self.0, bit);
Expand All @@ -20,3 +22,78 @@ fn set(x: u32, bit: u32) -> u32 {
fn test(x: u32, bit: u32) -> bool {
x & (1 << bit) != 0
}

#[inline]
pub(crate) fn detect() -> CpuInfo {
use core::sync::atomic::{AtomicU32, Ordering};

static CACHE: AtomicU32 = AtomicU32::new(0);
let mut info = CpuInfo(CACHE.load(Ordering::Relaxed));
if info.0 != 0 {
return info;
}
info.set(CpuInfo::INIT);
_detect(&mut info);
CACHE.store(info.0, Ordering::Relaxed);
info
}

#[cfg(target_arch = "aarch64")]
impl CpuInfo {
const HAS_LSE: u32 = 1;
}
#[cfg(target_arch = "aarch64")]
#[inline]
pub(crate) fn has_lse() -> bool {
#[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
{
// FEAT_LSE is statically available.
true
}
#[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
{
detect().test(CpuInfo::HAS_LSE)
}
}

#[cfg(target_arch = "x86_64")]
impl CpuInfo {
const HAS_CMPXCHG16B: u32 = 1;
const HAS_VMOVDQA_ATOMIC: u32 = 2;

#[allow(clippy::unused_self)]
#[inline]
pub(crate) fn has_cmpxchg16b(self) -> bool {
#[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
{
// CMPXCHG16B is statically available.
true
}
#[cfg(not(any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b"
)))]
{
self.test(CpuInfo::HAS_CMPXCHG16B)
}
}
#[inline]
pub(crate) fn has_vmovdqa_atomic(self) -> bool {
self.test(CpuInfo::HAS_VMOVDQA_ATOMIC)
}
}
/// Equivalent to `detect().has_cmpxchg16b()`, but avoids calling `detect()`
/// if CMPXCHG16B is statically available.
#[cfg(target_arch = "x86_64")]
#[inline]
pub(crate) fn has_cmpxchg16b() -> bool {
#[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
{
// CMPXCHG16B is statically available.
true
}
#[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))]
{
detect().has_cmpxchg16b()
}
}
66 changes: 4 additions & 62 deletions src/imp/atomic128/detect/x86_64.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,37 +14,7 @@ include!("common.rs");

#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::{
arch::x86_64::CpuidResult,
sync::atomic::{AtomicU32, Ordering},
};

impl CpuInfo {
const INIT: u32 = 0;
const HAS_CMPXCHG16B: u32 = 1;
const HAS_VMOVDQA_ATOMIC: u32 = 2;

#[allow(clippy::unused_self)]
#[inline]
pub(crate) fn has_cmpxchg16b(self) -> bool {
#[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
{
// CMPXCHG16B is statically available.
true
}
#[cfg(not(any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b"
)))]
{
self.test(CpuInfo::HAS_CMPXCHG16B)
}
}
#[inline]
pub(crate) fn has_vmovdqa_atomic(self) -> bool {
self.test(CpuInfo::HAS_VMOVDQA_ATOMIC)
}
}
use core::arch::x86_64::CpuidResult;

// Workaround for https://github.com/rust-lang/rust/issues/101346
// It is not clear if our use cases are affected, but we implement this just in case.
Expand Down Expand Up @@ -93,8 +63,7 @@ unsafe fn _vendor_id() -> [u8; 12] {
}

#[inline]
fn _cpuid(info: &mut CpuInfo) {
info.set(CpuInfo::INIT);
fn _detect(info: &mut CpuInfo) {
// Miri doesn't support inline assembly used in __cpuid
#[cfg(miri)]
{
Expand Down Expand Up @@ -136,33 +105,6 @@ fn _cpuid(info: &mut CpuInfo) {
}
}

#[inline]
pub(crate) fn cpuid() -> CpuInfo {
static CACHE: AtomicU32 = AtomicU32::new(0);
let mut info = CpuInfo(CACHE.load(Ordering::Relaxed));
if info.0 != 0 {
return info;
}
_cpuid(&mut info);
CACHE.store(info.0, Ordering::Relaxed);
info
}

/// Equivalent to `cpuid().has_cmpxchg16b()`, but avoids calling `cpuid()`
/// if CMPXCHG16B is statically available.
#[inline]
pub(crate) fn has_cmpxchg16b() -> bool {
#[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
{
// CMPXCHG16B is statically available.
true
}
#[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))]
{
cpuid().has_cmpxchg16b()
}
}

#[allow(clippy::undocumented_unsafe_blocks)]
#[cfg(test)]
mod tests {
Expand Down Expand Up @@ -196,9 +138,9 @@ mod tests {
assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), has_cmpxchg16b());
let vendor_id = unsafe { _vendor_id() };
if vendor_id == VENDOR_ID_INTEL || vendor_id == VENDOR_ID_AMD {
assert_eq!(std::is_x86_feature_detected!("avx"), cpuid().has_vmovdqa_atomic());
assert_eq!(std::is_x86_feature_detected!("avx"), detect().has_vmovdqa_atomic());
} else {
assert!(!cpuid().has_vmovdqa_atomic());
assert!(!detect().has_vmovdqa_atomic());
}
}
}
4 changes: 2 additions & 2 deletions src/imp/atomic128/x86_64.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
ifunc!(unsafe fn(src: *mut u128, order: Ordering) -> u128;
{
// Check CMPXCHG16B anyway to prevent mixing atomic and non-atomic access.
let cpuid = detect::cpuid();
let cpuid = detect::detect();
if cpuid.has_cmpxchg16b() && cpuid.has_vmovdqa_atomic() {
_atomic_load_vmovdqa
} else {
Expand Down Expand Up @@ -357,7 +357,7 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
ifunc!(unsafe fn(dst: *mut u128, val: u128, order: Ordering);
{
// Check CMPXCHG16B anyway to prevent mixing atomic and non-atomic access.
let cpuid = detect::cpuid();
let cpuid = detect::detect();
if cpuid.has_cmpxchg16b() && cpuid.has_vmovdqa_atomic() {
_atomic_store_vmovdqa
} else {
Expand Down

0 comments on commit de86109

Please sign in to comment.