Skip to content

Commit

Permalink
riscv: Support RMW when zaamo enabled (even when unsafe-assume-single…
Browse files Browse the repository at this point in the history 
…-core disabled)
  • Loading branch information
@taiki-e
taiki-e committed Sep 24, 2024
1 parent 6e6b5f9 commit fb8e656
Show file tree
Hide file tree
Showing 6 changed files with 304 additions and 62 deletions.
78 changes: 78 additions & 0 deletions src/cfgs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,17 @@ mod atomic_cas_macros {
macro_rules! cfg_no_atomic_cas {
($($tt:tt)*) => {};
}
// private
macro_rules! cfg_has_atomic_cas_or_amo32 {
($($tt:tt)*) => {
$($tt)*
};
}
macro_rules! cfg_has_atomic_cas_or_amo8 {
($($tt:tt)*) => {
$($tt)*
};
}
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
Expand Down Expand Up @@ -529,6 +540,73 @@ mod atomic_cas_macros {
$($tt)*
};
}
// private
#[cfg_attr(
any(target_arch = "riscv32", target_arch = "riscv64"),
cfg(not(any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo")))
)]
macro_rules! cfg_has_atomic_cas_or_amo32 {
($($tt:tt)*) => {};
}
#[cfg_attr(
any(target_arch = "riscv32", target_arch = "riscv64"),
cfg(not(any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo")))
)]
macro_rules! cfg_no_atomic_cas_or_amo32 {
($($tt:tt)*) => {
$($tt)*
};
}
#[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo"),
))]
macro_rules! cfg_has_atomic_cas_or_amo32 {
($($tt:tt)*) => {
$($tt)*
};
}
#[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo"),
))]
macro_rules! cfg_no_atomic_cas_or_amo32 {
($($tt:tt)*) => {};
}
#[cfg_attr(
any(target_arch = "riscv32", target_arch = "riscv64"),
cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))
)]
#[cfg_attr(target_arch = "bpf", allow(unused_macros))]
macro_rules! cfg_has_atomic_cas_or_amo8 {
($($tt:tt)*) => {};
}
#[cfg_attr(
any(target_arch = "riscv32", target_arch = "riscv64"),
cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))
)]
#[cfg_attr(target_arch = "bpf", allow(unused_macros))]
macro_rules! cfg_no_atomic_cas_or_amo8 {
($($tt:tt)*) => {
$($tt)*
};
}
#[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
any(target_feature = "zabha", portable_atomic_target_feature = "zabha"),
))]
macro_rules! cfg_has_atomic_cas_or_amo8 {
($($tt:tt)*) => {
$($tt)*
};
}
#[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
any(target_feature = "zabha", portable_atomic_target_feature = "zabha"),
))]
macro_rules! cfg_no_atomic_cas_or_amo8 {
($($tt:tt)*) => {};
}
}

// Check that all cfg_ macros work.
Expand Down
7 changes: 4 additions & 3 deletions src/imp/float.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,15 @@ macro_rules! atomic_float {
}
}

cfg_has_atomic_cas! {
cfg_has_atomic_cas_or_amo32! {
impl $atomic_type {
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn swap(&self, val: $float_type, order: Ordering) -> $float_type {
$float_type::from_bits(self.as_bits().swap(val.to_bits(), order))
}

cfg_has_atomic_cas! {
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
Expand Down Expand Up @@ -185,7 +186,7 @@ macro_rules! atomic_float {
pub(crate) fn fetch_min(&self, val: $float_type, order: Ordering) -> $float_type {
self.fetch_update_(order, |x| x.min(val))
}

} // cfg_has_atomic_cas!
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_neg(&self, order: Ordering) -> $float_type {
Expand All @@ -200,7 +201,7 @@ macro_rules! atomic_float {
$float_type::from_bits(self.as_bits().fetch_and(ABS_MASK, order))
}
}
} // cfg_has_atomic_cas!
} // cfg_has_atomic_cas_or_amo32!
};
}

Expand Down
65 changes: 65 additions & 0 deletions src/imp/riscv.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,22 @@ macro_rules! atomic_ptr {
macro_rules! atomic {
($atomic_type:ident, $value_type:ty, $asm_suffix:tt, $max:tt, $min:tt) => {
atomic_load_store!($atomic_type, $value_type, $asm_suffix);
#[cfg(any(
test,
portable_atomic_force_amo,
target_feature = "zaamo",
portable_atomic_target_feature = "zaamo",
))]
#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
impl_default_no_fetch_ops!($atomic_type, $value_type);
#[cfg(any(
test,
portable_atomic_force_amo,
target_feature = "zaamo",
portable_atomic_target_feature = "zaamo",
))]
#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
impl_default_bit_opts!($atomic_type, $value_type);
// There is no amo{sub,nand,neg}.
#[cfg(any(
test,
Expand Down Expand Up @@ -356,6 +372,14 @@ macro_rules! atomic {
pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {
self.fetch_xor(!0, order)
}
#[cfg(not(any(
portable_atomic_unsafe_assume_single_core,
feature = "critical-section",
)))]
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}

#[inline]
pub(crate) fn fetch_max(&self, val: $value_type, order: Ordering) -> $value_type {
Expand Down Expand Up @@ -434,6 +458,15 @@ macro_rules! atomic_sub_word {
portable_atomic_target_feature = "zaamo",
))]
#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
impl_default_bit_opts!($atomic_type, $value_type);
#[cfg(any(
test,
portable_atomic_force_amo,
target_feature = "zaamo",
portable_atomic_target_feature = "zaamo",
))]
#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
impl $atomic_type {
#[inline]
pub(crate) fn fetch_and(&self, val: $value_type, order: Ordering) -> $value_type {
Expand All @@ -447,6 +480,14 @@ macro_rules! atomic_sub_word {
let out: u32 = unsafe { atomic_rmw_amo!(and, dst, val, order, "w") };
srlw!(out, shift)
}
#[cfg(not(any(
portable_atomic_unsafe_assume_single_core,
feature = "critical-section",
)))]
#[inline]
pub(crate) fn and(&self, val: $value_type, order: Ordering) {
self.fetch_and(val, order);
}

#[inline]
pub(crate) fn fetch_or(&self, val: $value_type, order: Ordering) -> $value_type {
Expand All @@ -458,6 +499,14 @@ macro_rules! atomic_sub_word {
let out: u32 = unsafe { atomic_rmw_amo!(or, dst, val, order, "w") };
srlw!(out, shift)
}
#[cfg(not(any(
portable_atomic_unsafe_assume_single_core,
feature = "critical-section",
)))]
#[inline]
pub(crate) fn or(&self, val: $value_type, order: Ordering) {
self.fetch_or(val, order);
}

#[inline]
pub(crate) fn fetch_xor(&self, val: $value_type, order: Ordering) -> $value_type {
Expand All @@ -469,11 +518,27 @@ macro_rules! atomic_sub_word {
let out: u32 = unsafe { atomic_rmw_amo!(xor, dst, val, order, "w") };
srlw!(out, shift)
}
#[cfg(not(any(
portable_atomic_unsafe_assume_single_core,
feature = "critical-section",
)))]
#[inline]
pub(crate) fn xor(&self, val: $value_type, order: Ordering) {
self.fetch_xor(val, order);
}

#[inline]
pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {
self.fetch_xor(!0, order)
}
#[cfg(not(any(
portable_atomic_unsafe_assume_single_core,
feature = "critical-section",
)))]
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
}
};
}
Expand Down
Loading

0 comments on commit fb8e656

Please sign in to comment.