Skip to content

Commit

Permalink
Replace the use of C atomics with LLVM intrinsics
Browse files Browse the repository at this point in the history
  • Loading branch information
@lorentey
lorentey committed Mar 20, 2023
1 parent ca8a8fc commit c48c132
Show file tree
Hide file tree
Showing 19 changed files with 10,139 additions and 4,958 deletions.
9 changes: 8 additions & 1 deletion Package.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,16 @@ let package = Package(
"CMakeLists.txt"
]
),
.target(
name: "_AtomicBuiltins",
exclude: [
"CMakeLists.txt"
],
swiftSettings: [.unsafeFlags(["-Xfrontend", "-parse-stdlib"])]
),
.target(
name: "Atomics",
dependencies: ["_AtomicsShims"],
dependencies: ["_AtomicsShims", "_AtomicBuiltins"],
exclude: [
"CMakeLists.txt",
"HighLevelTypes.swift.gyb",
Expand Down
154 changes: 50 additions & 104 deletions Sources/Atomics/AtomicBool.swift.gyb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
//
// This source file is part of the Swift Atomics open source project
//
// Copyright (c) 2020-2021 Apple Inc. and the Swift project authors
// Copyright (c) 2020-2023 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
Expand All @@ -12,41 +12,49 @@

%{
from gyb_utils import (
autogenerated_warning, loadOrderings, storeOrderings, updateOrderings,
boolOperations, lowerFirst, argLabel, shimOrders)
autogenerated_warning, boolOperations, lowerFirst, argLabel)
}%
${autogenerated_warning()}

import _AtomicsShims

extension Bool: AtomicValue {
@frozen
public struct AtomicRepresentation {
public typealias Value = Bool

@usableFromInline
internal var _storage: _AtomicBoolStorage
internal var _storage: Int8

@inline(__always) @_alwaysEmitIntoClient
public init(_ value: Bool) {
_storage = _sa_prepare_Bool(value)
_storage = value._atomicValue
}

@inline(__always) @_alwaysEmitIntoClient
public func dispose() -> Value {
_sa_dispose_Bool(_storage)
_storage._atomicBoolValue
}
}

@_alwaysEmitIntoClient @inline(__always)
internal var _atomicValue: Int8 {
self ? 1 : 0
}
}

extension Bool.AtomicRepresentation {
extension Int8 {
@_alwaysEmitIntoClient @inline(__always)
internal var _atomicBoolValue: Bool {
(self & 1) != 0
}
}

extension UnsafeMutablePointer
where Pointee == Bool.AtomicRepresentation {
@_transparent @_alwaysEmitIntoClient
@usableFromInline
static func _extract(
_ ptr: UnsafeMutablePointer<Self>
) -> UnsafeMutablePointer<_AtomicBoolStorage> {
internal var _extract: UnsafeMutablePointer<Int8> {
// `Self` is layout-compatible with its only stored property.
return UnsafeMutableRawPointer(ptr)
.assumingMemoryBound(to: _AtomicBoolStorage.self)
UnsafeMutableRawPointer(self).assumingMemoryBound(to: Int8.self)
}
}

Expand All @@ -57,14 +65,7 @@ extension Bool.AtomicRepresentation: AtomicStorage {
at pointer: UnsafeMutablePointer<Bool.AtomicRepresentation>,
ordering: AtomicLoadOrdering
) -> Bool {
switch ordering {
% for (swiftOrder, shimOrder) in loadOrderings:
case .${swiftOrder}:
return _sa_load_${shimOrder}_Bool(_extract(pointer))
% end
default:
fatalError("Unsupported ordering")
}
pointer._extract._atomicLoad(ordering: ordering)._atomicBoolValue
}

@_semantics("atomics.requires_constant_orderings")
Expand All @@ -74,14 +75,7 @@ extension Bool.AtomicRepresentation: AtomicStorage {
at pointer: UnsafeMutablePointer<Bool.AtomicRepresentation>,
ordering: AtomicStoreOrdering
) {
switch ordering {
% for (swiftOrder, shimOrder) in storeOrderings:
case .${swiftOrder}:
_sa_store_${shimOrder}_Bool(_extract(pointer), desired)
% end
default:
fatalError("Unsupported ordering")
}
pointer._extract._atomicStore(desired._atomicValue, ordering: ordering)
}

@_semantics("atomics.requires_constant_orderings")
Expand All @@ -91,14 +85,9 @@ extension Bool.AtomicRepresentation: AtomicStorage {
at pointer: UnsafeMutablePointer<Bool.AtomicRepresentation>,
ordering: AtomicUpdateOrdering
) -> Bool {
switch ordering {
% for (swiftOrder, shimOrder, _) in updateOrderings:
case .${swiftOrder}:
return _sa_exchange_${shimOrder}_Bool(_extract(pointer), desired)
% end
default:
fatalError("Unsupported ordering")
}
pointer._extract._atomicExchange(
desired._atomicValue, ordering: ordering
)._atomicBoolValue
}

@_semantics("atomics.requires_constant_orderings")
Expand All @@ -109,19 +98,11 @@ extension Bool.AtomicRepresentation: AtomicStorage {
at pointer: UnsafeMutablePointer<Bool.AtomicRepresentation>,
ordering: AtomicUpdateOrdering
) -> (exchanged: Bool, original: Bool) {
var expected = expected
let exchanged: Bool
switch ordering {
% for (swiftOrder, shimOrder, failOrder) in updateOrderings:
case .${swiftOrder}:
exchanged = _sa_cmpxchg_strong_${shimOrder}_${failOrder}_Bool(
_extract(pointer),
&expected, desired)
% end
default:
fatalError("Unsupported ordering")
}
return (exchanged, expected)
let r = pointer._extract._atomicCompareExchange(
expected: expected._atomicValue,
desired: desired._atomicValue,
ordering: ordering)
return (r.exchanged, r.original._atomicBoolValue)
}

@_semantics("atomics.requires_constant_orderings")
Expand All @@ -133,26 +114,12 @@ extension Bool.AtomicRepresentation: AtomicStorage {
successOrdering: AtomicUpdateOrdering,
failureOrdering: AtomicLoadOrdering
) -> (exchanged: Bool, original: Bool) {
var expected = expected
let exchanged: Bool
// FIXME: stdatomic.h (and LLVM underneath) doesn't support
// arbitrary ordering combinations yet, so upgrade the success
// ordering when necessary so that it is at least as "strong" as
// the failure case.
switch (successOrdering, failureOrdering) {
% for (swiftSuccess, shimSuccess, _) in updateOrderings:
% for (swiftFailure, shimFailure) in loadOrderings:
case (.${swiftSuccess}, .${swiftFailure}):
exchanged = _sa_cmpxchg_strong_${shimOrders(shimSuccess, shimFailure)}_Bool(
_extract(pointer),
&expected,
desired)
% end
% end
default:
fatalError("Unsupported ordering")
}
return (exchanged, expected)
let r = pointer._extract._atomicCompareExchange(
expected: expected._atomicValue,
desired: desired._atomicValue,
successOrdering: successOrdering,
failureOrdering: failureOrdering)
return (r.exchanged, r.original._atomicBoolValue)
}

@_semantics("atomics.requires_constant_orderings")
Expand All @@ -164,34 +131,20 @@ extension Bool.AtomicRepresentation: AtomicStorage {
successOrdering: AtomicUpdateOrdering,
failureOrdering: AtomicLoadOrdering
) -> (exchanged: Bool, original: Bool) {
var expected = expected
let exchanged: Bool
// FIXME: stdatomic.h (and LLVM underneath) doesn't support
// arbitrary ordering combinations yet, so upgrade the success
// ordering when necessary so that it is at least as "strong" as
// the failure case.
switch (successOrdering, failureOrdering) {
% for (swiftSuccess, shimSuccess, _) in updateOrderings:
% for (swiftFailure, shimFailure) in loadOrderings:
case (.${swiftSuccess}, .${swiftFailure}):
exchanged = _sa_cmpxchg_weak_${shimOrders(shimSuccess, shimFailure)}_Bool(
_extract(pointer),
&expected,
desired)
% end
% end
default:
fatalError("Unsupported ordering")
}
return (exchanged, expected)
let r = pointer._extract._atomicWeakCompareExchange(
expected: expected._atomicValue,
desired: desired._atomicValue,
successOrdering: successOrdering,
failureOrdering: failureOrdering)
return (r.exchanged, r.original._atomicBoolValue)
}
}


// MARK: - Additional operations

extension Bool.AtomicRepresentation {
% for (name, cname, op, label, doc) in boolOperations:
% for (name, iname, op, label, doc) in boolOperations:
/// Perform an atomic ${doc} operation on the value referenced by
/// `pointer` and return the original value, applying the specified memory
/// ordering.
Expand All @@ -208,23 +161,16 @@ extension Bool.AtomicRepresentation {
at pointer: UnsafeMutablePointer<Self>,
ordering: AtomicUpdateOrdering
) -> Value {
switch ordering {
% for (swiftOrder, shimOrder, _) in updateOrderings:
case .${swiftOrder}:
return _sa_fetch_${cname}_${shimOrder}_Bool(
_extract(pointer),
operand)
% end
default:
fatalError("Unsupported ordering")
}
pointer._extract._atomicLoadThen${iname}(
${argLabel(label)}operand._atomicValue, ordering: ordering
)._atomicBoolValue
}
% end
}

% for construct in ["UnsafeAtomic", "ManagedAtomic"]:
extension ${construct} where Value == Bool {
% for (name, cname, op, label, doc) in boolOperations:
% for (name, iname, op, label, doc) in boolOperations:
/// Perform an atomic ${doc} operation and return the original value, applying
/// the specified memory ordering.
///
Expand All @@ -246,7 +192,7 @@ extension ${construct} where Value == Bool {
}

extension ${construct} where Value == Bool {
% for (name, cname, op, label, doc) in boolOperations:
% for (name, iname, op, label, doc) in boolOperations:
/// Perform an atomic ${doc} operation and return the original value, applying
/// the specified memory ordering.
///
Expand Down
10 changes: 5 additions & 5 deletions Sources/Atomics/AtomicMemoryOrderings.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
//
//===----------------------------------------------------------------------===//

import _AtomicsShims
import _AtomicBuiltins

/// Specifies the memory ordering semantics of an atomic load operation.
@frozen
Expand Down Expand Up @@ -315,10 +315,10 @@ public func atomicMemoryFence(
) {
switch ordering {
case .relaxed: break
case .acquiring: _sa_thread_fence_acquire()
case .releasing: _sa_thread_fence_release()
case .acquiringAndReleasing: _sa_thread_fence_acq_rel()
case .sequentiallyConsistent: _sa_thread_fence_seq_cst()
case .acquiring: _atomicAcquiringMemoryFence()
case .releasing: _atomicReleasingMemoryFence()
case .acquiringAndReleasing: _atomicAcquiringAndReleasingMemoryFence()
case .sequentiallyConsistent: _atomicSequentiallyConsistentMemoryFence()
default: fatalError()
}
}
19 changes: 1 addition & 18 deletions Sources/Atomics/AtomicStrongReference.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
//
//===----------------------------------------------------------------------===//

import _AtomicsShims
import _AtomicBuiltins

/// A class type that supports atomic strong references.
///
Expand Down Expand Up @@ -74,23 +74,6 @@ where
@inlinable @inline(__always)
internal var _concurrencyWindow: Int { 20 }

extension Unmanaged {
internal func retain(by delta: Int) {
_sa_retain_n(toOpaque(), UInt32(delta))
}

internal func release(by delta: Int) {
_sa_release_n(toOpaque(), UInt32(delta))
}
}

extension Unmanaged {
fileprivate static func passRetained(_ instance: __owned Instance?) -> Self? {
guard let instance = instance else { return nil }
return .passRetained(instance)
}
}

extension DoubleWord {
fileprivate init(_raw: UnsafeRawPointer?, readers: Int, version: Int) {
let r = UInt(bitPattern: readers) & Self._readersMask
Expand Down
Loading

0 comments on commit c48c132

Please sign in to comment.