riscv32: Support 64-bit atomics (Zacas extension)

taiki-e · Sep 18, 2024 · 6df3cd1 · 6df3cd1
1 parent 5d38a3f
commit 6df3cd1
Show file tree

Hide file tree

Showing 13 changed files with 859 additions and 38 deletions.
diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt
@@ -46,6 +46,8 @@ DWCAS
 elems
 espup
 exynos
+fild
+fistp
 getauxval
 getisax
 getpid
@@ -70,6 +72,7 @@ ldclrpa
 ldclrpal
 ldclrpl
 ldiapp
+ldrexd
 ldsetp
 ldsetpa
 ldsetpal
@@ -101,6 +104,8 @@ minu
 mipsn
 miscompiles
 mmfr
+movlps
+movq
 mpidr
 mstatus
 mvfr
@@ -150,6 +155,7 @@ stilp
 stlxp
 stpq
 stqcx
+strexd
 stxp
 subarch
 subc

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -259,8 +259,6 @@ jobs:
             target: riscv64gc-unknown-linux-gnu
           - rust: '1.59' # LLVM 13
             target: riscv64gc-unknown-linux-gnu
-          - rust: '1.73' # LLVM 17 (oldest version we can use experimental-zacas on this target)
-            target: riscv64gc-unknown-linux-gnu
           - rust: stable
             target: riscv64gc-unknown-linux-gnu
           - rust: nightly
@@ -317,6 +315,9 @@ jobs:
       # TODO: LLVM bug: Undefined temporary symbol error when building std.
       - run: printf 'RELEASE=--release\n' >>"${GITHUB_ENV}"
         if: startsWith(matrix.target, 'mips-') || startsWith(matrix.target, 'mipsel-')
+      # for serde
+      - run: printf '%s\n' "RUSTFLAGS=${RUSTFLAGS} --cfg no_diagnostic_namespace" >>"${GITHUB_ENV}"
+        if: matrix.rust == 'nightly-2024-02-13'
 
       - run: tools/test.sh -vv ${TARGET:-} ${DOCTEST_XCOMPILE:-} ${BUILD_STD:-} ${RELEASE:-}
       # We test doctest only once with the default build conditions because doctest is slow. Both api-test
@@ -388,21 +389,21 @@ jobs:
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-cpu=pwr8
           RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-cpu=pwr8
         if: startsWith(matrix.target, 'powerpc64-')
-      # riscv64 +zabha
+      # riscv +zabha
       - run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
         env:
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+zabha
           RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+zabha
           QEMU_CPU: max
         # TODO: cranelift doesn't support cfg(target_feature): https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400
-        if: startsWith(matrix.target, 'riscv64') && !contains(matrix.flags, 'codegen-backend=cranelift')
-      # riscv64 +experimental-zacas
+        if: startsWith(matrix.target, 'riscv') && !contains(matrix.flags, 'codegen-backend=cranelift')
+      # riscv +experimental-zacas
       - run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
         env:
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+experimental-zacas
           RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+experimental-zacas
         # TODO: cranelift doesn't support cfg(target_feature): https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400
-        if: startsWith(matrix.target, 'riscv64') && !contains(matrix.flags, 'codegen-backend=cranelift')
+        if: startsWith(matrix.target, 'riscv') && !contains(matrix.flags, 'codegen-backend=cranelift')
       # s390x z196 (arch9)
       - run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
         env:

diff --git a/src/cfgs.rs b/src/cfgs.rs
@@ -154,6 +154,36 @@ mod atomic_32_macros {
         ),
         target_has_atomic = "64",
         not(any(target_pointer_width = "16", target_pointer_width = "32")),
+        all(
+            target_arch = "riscv32",
+            not(any(miri, portable_atomic_sanitize_thread)),
+            not(portable_atomic_no_asm),
+            any(
+                target_feature = "experimental-zacas",
+                portable_atomic_target_feature = "experimental-zacas",
+                // TODO(riscv)
+                // all(
+                //     feature = "fallback",
+                //     not(portable_atomic_no_outline_atomics),
+                //     any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
+                //     any(
+                //         all(
+                //             target_os = "linux",
+                //             any(
+                //                 target_env = "gnu",
+                //                 all(
+                //                     any(target_env = "musl", target_env = "ohos"),
+                //                     not(target_feature = "crt-static"),
+                //                 ),
+                //                 portable_atomic_outline_atomics,
+                //             ),
+                //         ),
+                //         target_os = "android",
+                //     ),
+                //     not(any(miri, portable_atomic_sanitize_thread)),
+                // ),
+            ),
+        ),
     ))
 )]
 #[macro_use]
@@ -201,6 +231,36 @@ mod atomic_64_macros {
         ),
         target_has_atomic = "64",
         not(any(target_pointer_width = "16", target_pointer_width = "32")),
+        all(
+            target_arch = "riscv32",
+            not(any(miri, portable_atomic_sanitize_thread)),
+            not(portable_atomic_no_asm),
+            any(
+                target_feature = "experimental-zacas",
+                portable_atomic_target_feature = "experimental-zacas",
+                // TODO(riscv)
+                // all(
+                //     feature = "fallback",
+                //     not(portable_atomic_no_outline_atomics),
+                //     any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
+                //     any(
+                //         all(
+                //             target_os = "linux",
+                //             any(
+                //                 target_env = "gnu",
+                //                 all(
+                //                     any(target_env = "musl", target_env = "ohos"),
+                //                     not(target_feature = "crt-static"),
+                //                 ),
+                //                 portable_atomic_outline_atomics,
+                //             ),
+                //         ),
+                //         target_os = "android",
+                //     ),
+                //     not(any(miri, portable_atomic_sanitize_thread)),
+                // ),
+            ),
+        ),
     )))
 )]
 #[macro_use]
@@ -247,11 +307,11 @@ mod atomic_64_macros {
             any(
                 target_feature = "experimental-zacas",
                 portable_atomic_target_feature = "experimental-zacas",
-                // TODO(riscv64)
+                // TODO(riscv)
                 // all(
                 //     feature = "fallback",
                 //     not(portable_atomic_no_outline_atomics),
-                //     any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
+                //     any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
                 //     any(
                 //         all(
                 //             target_os = "linux",
@@ -366,11 +426,11 @@ mod atomic_128_macros {
             any(
                 target_feature = "experimental-zacas",
                 portable_atomic_target_feature = "experimental-zacas",
-                // TODO(riscv64)
+                // TODO(riscv)
                 // all(
                 //     feature = "fallback",
                 //     not(portable_atomic_no_outline_atomics),
-                //     any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
+                //     any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
                 //     any(
                 //         all(
                 //             target_os = "linux",

diff --git a/src/imp/atomic128/riscv64.rs b/src/imp/atomic128/riscv64.rs
@@ -18,8 +18,6 @@ Generated asm:
 - riscv64 (+experimental-zacas) https://godbolt.org/z/5Kc17T1W8
 */
 
-// TODO: 64-bit atomic using amocas.d for riscv32
-
 include!("macros.rs");
 
 // TODO
@@ -31,7 +29,7 @@ include!("macros.rs");
 // See detect/auxv.rs for more.
 #[cfg(test)] // TODO
 #[cfg(not(portable_atomic_no_outline_atomics))]
-#[cfg(any(test, portable_atomic_outline_atomics))] // TODO(riscv64): currently disabled by default
+#[cfg(any(test, portable_atomic_outline_atomics))] // TODO(riscv): currently disabled by default
 #[cfg(any(
     test,
     not(any(

diff --git a/src/imp/atomic64/README.md b/src/imp/atomic64/README.md
@@ -1,3 +1,34 @@
 # Implementation of 64-bit atomics on 32-bit architectures
 
 (See the [`atomic128` module](../atomic128) for 128-bit atomics on 64-bit architectures.)
+
+## 64-bit atomics instructions
+
+Here is the table of targets that support 64-bit atomics and the instructions used:
+
+| target_arch | load | store | CAS | RMW | note |
+| ----------- | ---- | ----- | --- | --- | ---- |
+| x86 | cmpxchg8b or fild or movlps or movq | cmpxchg8b or fistp or movlps | cmpxchg8b | cmpxchg8b | provided by `core::sync::atomic` |
+| arm | ldrexd | ldrexd/strexd | ldrexd/strexd | ldrexd/strexd | provided by `core::sync::atomic` for Armv6+, otherwise provided by us for Linux/Android using kuser_cmpxchg64 (see arm_linux.rs for more) |
+| riscv32 | amocas.d | amocas.d | amocas.d | amocas.d | Experimental. Requires experimental-zacas target feature. Currently compile-time detection only due to LLVM marking it as experimental. <br> Requires 1.82+ (LLVM 19+) |
+
+If `core::sync::atomic` provides 64-bit atomics, we use them.
+On compiler versions or platforms where these are not supported, the fallback implementation is used.
+
+## Run-time CPU feature detection
+
+[detect](../detect) module has run-time CPU feature detection implementations.
+
+Here is the table of targets that support run-time CPU feature detection and the instruction or API used:
+
+| target_arch | target_os/target_env | instruction/API | features | note |
+| ----------- | -------------------- | --------------- | -------- | ---- |
+| riscv32     | linux                | riscv_hwprobe   | all      | Currently only used in tests due to LLVM marking zacas as experimental |
+
+Run-time detection is enabled by default on most targets and can be disabled with `--cfg portable_atomic_no_outline_atomics`.
+
+On some targets, run-time detection is disabled by default mainly for compatibility with older versions of operating systems or incomplete build environments, and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled, `*_no_*` cfg is preferred.)
+
+For targets not included in the above table, run-time detection is always disabled and works the same as when `--cfg portable_atomic_no_outline_atomics` is set.
+
+See also [docs on `portable_atomic_no_outline_atomics`](https://github.com/taiki-e/portable-atomic/blob/HEAD/README.md#optional-cfg-no-outline-atomics) in the top-level readme.