diff --git a/crates/spirv-std/src/arch.rs b/crates/spirv-std/src/arch.rs index 28816cea6d..60fe0a563a 100644 --- a/crates/spirv-std/src/arch.rs +++ b/crates/spirv-std/src/arch.rs @@ -11,12 +11,14 @@ use crate::{ #[cfg(target_arch = "spirv")] use core::arch::asm; +mod atomics; mod barrier; mod demote_to_helper_invocation_ext; mod derivative; mod primitive; mod ray_tracing; +pub use atomics::*; pub use barrier::*; pub use demote_to_helper_invocation_ext::*; pub use derivative::*; @@ -226,29 +228,6 @@ pub fn signed_max(a: T, b: T) -> T { unsafe { call_glsl_op_with_ints::<_, 42>(a, b) } } -/// Atomically increment an integer and return the old value. -#[spirv_std_macros::gpu_only] -#[doc(alias = "OpAtomicIIncrement")] -pub unsafe fn atomic_i_increment( - ptr: &mut I, -) -> I { - let mut old = I::default(); - - asm! { - "%u32 = OpTypeInt 32 0", - "%scope = OpConstant %u32 {scope}", - "%semantics = OpConstant %u32 {semantics}", - "%old = OpAtomicIIncrement _ {ptr} %scope %semantics", - "OpStore {old} %old", - scope = const SCOPE, - semantics = const SEMANTICS, - ptr = in(reg) ptr, - old = in(reg) &mut old, - } - - old -} - /// Index into an array without bounds checking. /// /// The main purpose of this trait is to work around the fact that the regular `get_unchecked*` diff --git a/crates/spirv-std/src/arch/atomics.rs b/crates/spirv-std/src/arch/atomics.rs new file mode 100644 index 0000000000..bb0f75992e --- /dev/null +++ b/crates/spirv-std/src/arch/atomics.rs @@ -0,0 +1,611 @@ +#[cfg(target_arch = "spirv")] +use core::arch::asm; + +use crate::{ + float::Float, + integer::{Integer, SignedInteger, UnsignedInteger}, + number::Number, +}; + +/// Atomically load through `ptr` using the given `SEMANTICS`. All subparts of +/// the value that is loaded are read atomically with respect to all other +/// atomic accesses to it within `SCOPE`. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicLoad")] +#[inline] +pub unsafe fn atomic_load(ptr: &N) -> N { + let mut result = N::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%result = OpAtomicLoad _ {ptr} %scope %semantics", + "OpStore {result} %result", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + result = in(reg) &mut result + } + + result +} + +/// Atomically store through `ptr` using the given `SEMANTICS`. All subparts of +/// `value` are written atomically with respect to all other atomic accesses to +/// it within `SCOPE`. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicStore")] +#[inline] +pub unsafe fn atomic_store( + ptr: &mut N, + value: N, +) { + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "OpAtomicStore {ptr} %scope %semantics %value", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + value = in(reg) &value + } +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within `SCOPE` to the same location: +/// +/// 1. Load through `ptr` to get the original value, +/// 2. Get a new value from copying `value`, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicExchange")] +#[inline] +pub unsafe fn atomic_exchange( + ptr: &mut N, + value: N, +) -> N { + let mut old = N::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicExchange _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within `SCOPE` to the same location: +/// +/// 1. Load through `ptr` to get the original value +/// 2. Get a new value from `value` only if the original value equals +/// `comparator`, and +/// 3. Store the new value back through `ptr`, only if the original value +/// equaled `comparator`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicCompareExchange")] +#[inline] +pub unsafe fn atomic_compare_exchange< + I: Integer, + const SCOPE: u32, + const EQUAL: u32, + const UNEQUAL: u32, +>( + ptr: &mut I, + value: I, + comparator: I, +) -> I { + let mut old = I::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%equal = OpConstant %u32 {equal}", + "%unequal = OpConstant %u32 {unequal}", + "%value = OpLoad _ {value}", + "%comparator = OpLoad _ {comparator}", + "%old = OpAtomicCompareExchange _ {ptr} %scope %equal %unequal %value %comparator", + "OpStore {old} %old", + scope = const SCOPE, + equal = const EQUAL, + unequal = const UNEQUAL, + ptr = in(reg) ptr, + value = in(reg) &value, + comparator = in(reg) &comparator, + old = in(reg) &mut old, + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within `SCOPE` to the same location: +/// +/// 1. Load through `ptr` to get an original value, +/// 2. Get a new value through integer addition of 1 to original value, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicIIncrement")] +#[inline] +pub unsafe fn atomic_i_increment( + ptr: &mut I, +) -> I { + let mut old = I::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%old = OpAtomicIIncrement _ {ptr} %scope %semantics", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within `SCOPE` to the same location: +/// +/// 1) load through `ptr` to get an original value, +/// 2) get a new value through integer subtraction of 1 from original value, and +/// 3) store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicIDecrement")] +#[inline] +pub unsafe fn atomic_i_decrement( + ptr: &mut I, +) -> I { + let mut old = I::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%old = OpAtomicIDecrement _ {ptr} %scope %semantics", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within `SCOPE` to the same location: +/// +/// 1) load through `ptr` to get an original value, +/// 2) get a new value by integer addition of original value and `value`, and +/// 3) store the new value back through `ptr`. +/// +/// The result is the Original Value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicIAdd")] +#[inline] +pub unsafe fn atomic_i_add( + ptr: &mut I, + value: I, +) -> I { + let mut old = I::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicIAdd _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within `SCOPE` to the same location: +/// +/// 1) load through `ptr` to get an original value, +/// 2) get a new value by integer subtraction of original value and `value`, and +/// 3) store the new value back through `ptr`. +/// +/// The result is the Original Value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicISub")] +#[inline] +pub unsafe fn atomic_i_sub( + ptr: &mut I, + value: I, +) -> I { + let mut old = I::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicISub _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within Scope to the same location: +/// +/// 1. Load through `ptr` to get an original value, +/// 2. Get a new value by finding the smallest signed integer of original value +/// and `value`, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicSMin")] +#[inline] +pub unsafe fn atomic_s_min( + ptr: &mut S, + value: S, +) -> S { + let mut old = S::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicSMin _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within Scope to the same location: +/// +/// 1. Load through `ptr` to get an original value, +/// 2. Get a new value by finding the smallest unsigned integer of original +/// value and `value`, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicUMin")] +#[inline] +pub unsafe fn atomic_u_min( + ptr: &mut U, + value: U, +) -> U { + let mut old = U::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicUMin _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within Scope to the same location: +/// +/// 1. Load through `ptr` to get an original value, +/// 2. Get a new value by finding the largest signed integer of original value +/// and `value`, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicSMax")] +#[inline] +pub unsafe fn atomic_s_max( + ptr: &mut S, + value: S, +) -> S { + let mut old = S::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicSMax _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within Scope to the same location: +/// +/// 1. Load through `ptr` to get an original value, +/// 2. Get a new value by finding the largest unsigned integer of original +/// value and `value`, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicUMax")] +#[inline] +pub unsafe fn atomic_u_max( + ptr: &mut U, + value: U, +) -> U { + let mut old = U::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicUMax _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within Scope to the same location: +/// +/// 1. Load through `ptr` to get an original value, +/// 2. Get a new value by the bitwise AND of the original value and `value`, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicAnd")] +#[inline] +pub unsafe fn atomic_and( + ptr: &mut I, + value: I, +) -> I { + let mut old = I::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicAnd _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within Scope to the same location: +/// +/// 1. Load through `ptr` to get an original value, +/// 2. Get a new value by the bitwise OR of the original value and `value`, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicOr")] +#[inline] +pub unsafe fn atomic_or( + ptr: &mut I, + value: I, +) -> I { + let mut old = I::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicOr _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within Scope to the same location: +/// +/// 1. Load through `ptr` to get an original value, +/// 2. Get a new value by the bitwise XOR of the original value and `value`, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicXor")] +#[inline] +pub unsafe fn atomic_xor( + ptr: &mut I, + value: I, +) -> I { + let mut old = I::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicXor _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within Scope to the same location: +/// +/// 1. Load through `ptr` to get an original value, +/// 2. Get a new value by finding the smallest signed integer of original value +/// and `value`, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicFMinEXT")] +#[inline] +pub unsafe fn atomic_f_min( + ptr: &mut F, + value: F, +) -> F { + let mut old = F::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicFMinEXT _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within Scope to the same location: +/// +/// 1. Load through `ptr` to get an original value, +/// 2. Get a new value by finding the largest signed integer of original value +/// and `value`, and +/// 3. Store the new value back through `ptr`. +/// +/// The result is the original value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicFMaxEXT")] +#[inline] +pub unsafe fn atomic_f_max( + ptr: &mut F, + value: F, +) -> F { + let mut old = F::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicFMaxEXT _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} + +/// Perform the following steps atomically with respect to any other atomic +/// accesses within `SCOPE` to the same location: +/// +/// 1) load through `ptr` to get an original value, +/// 2) get a new value by integer addition of original value and `value`, and +/// 3) store the new value back through `ptr`. +/// +/// The result is the Original Value. +#[spirv_std_macros::gpu_only] +#[doc(alias = "OpAtomicFAddEXT")] +#[inline] +pub unsafe fn atomic_f_add( + ptr: &mut F, + value: F, +) -> F { + let mut old = F::default(); + + asm! { + "%u32 = OpTypeInt 32 0", + "%scope = OpConstant %u32 {scope}", + "%semantics = OpConstant %u32 {semantics}", + "%value = OpLoad _ {value}", + "%old = OpAtomicFMaxEXT _ {ptr} %scope %semantics %value", + "OpStore {old} %old", + scope = const SCOPE, + semantics = const SEMANTICS, + ptr = in(reg) ptr, + old = in(reg) &mut old, + value = in(reg) &value + } + + old +} diff --git a/crates/spirv-std/src/lib.rs b/crates/spirv-std/src/lib.rs index 4101a4c35d..5b2e9a7681 100644 --- a/crates/spirv-std/src/lib.rs +++ b/crates/spirv-std/src/lib.rs @@ -102,6 +102,7 @@ pub mod float; pub mod image; pub mod integer; pub mod memory; +pub mod number; pub mod ray_tracing; mod runtime_array; mod sampler; diff --git a/crates/spirv-std/src/number.rs b/crates/spirv-std/src/number.rs new file mode 100644 index 0000000000..20a195da1e --- /dev/null +++ b/crates/spirv-std/src/number.rs @@ -0,0 +1,15 @@ +//! Traits and helper functions related to numbers. + +/// Abstract trait representing a SPIR-V integer or floating-point type. +pub trait Number: crate::scalar::Scalar {} + +impl Number for u8 {} +impl Number for u16 {} +impl Number for u32 {} +impl Number for u64 {} +impl Number for i8 {} +impl Number for i16 {} +impl Number for i32 {} +impl Number for i64 {} +impl Number for f32 {} +impl Number for f64 {} diff --git a/tests/ui/arch/debug_printf_type_checking.stderr b/tests/ui/arch/debug_printf_type_checking.stderr index f719d0abaa..d422023e6b 100644 --- a/tests/ui/arch/debug_printf_type_checking.stderr +++ b/tests/ui/arch/debug_printf_type_checking.stderr @@ -100,9 +100,9 @@ error[E0277]: the trait bound `{float}: Vector` is not satisfied > and 9 others note: required by a bound in `debug_printf_assert_is_vector` - --> $SPIRV_STD_SRC/lib.rs:144:8 + --> $SPIRV_STD_SRC/lib.rs:145:8 | -144 | V: crate::vector::Vector, +145 | V: crate::vector::Vector, | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `debug_printf_assert_is_vector` error[E0308]: mismatched types