Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Atomic operations on Pointers #37683

Closed
wants to merge 4 commits into from
Closed

Atomic operations on Pointers #37683

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7b011ba
use atomicrmw fadd and fsub instead of cmpxchg (atomic_cas) where app…
chethega Sep 21, 2020
6b10b98
add atomic op support to Ptr{T}
chethega Sep 21, 2020
ed4b6e1
fixup stupid styntax mistakes
chethega Sep 22, 2020
3374bee
fixup macro for Float16: atomicrmw fadd only supports float and double
chethega Sep 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 33 additions & 23 deletions base/atomics.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,27 +350,28 @@ import ..Base.gc_alignment
# All atomic operations have acquire and/or release semantics, depending on
# whether the load or store values. Most of the time, this is what one wants
# anyway, and it's only moderately expensive on most hardware.
for typ in atomictypes
for typ in atomictypes, container in [Atomic, Ptr]
lt = llvmtypes[typ]
ilt = llvmtypes[inttype(typ)]
rt = "$lt, $lt*"
irt = "$ilt, $ilt*"
@eval getindex(x::Atomic{$typ}) =
llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = load atomic $rt %ptr acquire, align $(gc_alignment(typ))
ret $lt %rv
""", $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x))
@eval setindex!(x::Atomic{$typ}, v::$typ) =
llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
store atomic $lt %1, $lt* %ptr release, align $(gc_alignment(typ))
ret void
""", Cvoid, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v)

if container == Atomic
@eval getindex(x::Atomic{$typ}) =
llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = load atomic $rt %ptr acquire, align $(gc_alignment(typ))
ret $lt %rv
""", $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x))
@eval setindex!(x::Atomic{$typ}, v::$typ) =
llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
store atomic $lt %1, $lt* %ptr release, align $(gc_alignment(typ))
ret void
""", Cvoid, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v)
end
# Note: atomic_cas! succeeded (i.e. it stored "new") if and only if the result is "cmp"
if typ <: Integer
@eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) =
@eval atomic_cas!(x::$container{$typ}, cmp::$typ, new::$typ) =
llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 acq_rel acquire
Expand All @@ -379,7 +380,7 @@ for typ in atomictypes
""", $typ, Tuple{Ptr{$typ},$typ,$typ},
unsafe_convert(Ptr{$typ}, x), cmp, new)
else
@eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) =
@eval atomic_cas!(x::$container{$typ}, cmp::$typ, new::$typ) =
llvmcall($"""
%iptr = inttoptr i$WORD_SIZE %0 to $ilt*
%icmp = bitcast $lt %1 to $ilt
Expand All @@ -402,31 +403,39 @@ for typ in atomictypes
end
if rmwop in arithmetic_ops && !(typ <: ArithmeticTypes) continue end
if typ <: Integer
@eval $fn(x::Atomic{$typ}, v::$typ) =
@eval $fn(x::$container{$typ}, v::$typ) =
llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 acq_rel
ret $lt %rv
""", $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v)
else
rmwop === :xchg || continue
@eval $fn(x::Atomic{$typ}, v::$typ) =
elseif rmwop === :xchg
@eval $fn(x::$container{$typ}, v::$typ) =
llvmcall($"""
%iptr = inttoptr i$WORD_SIZE %0 to $ilt*
%ival = bitcast $lt %1 to $ilt
%irv = atomicrmw $rmw $ilt* %iptr, $ilt %ival acq_rel
%rv = bitcast $ilt %irv to $lt
ret $lt %rv
""", $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v)
elseif (rmwop === :add || rmwop === :sub) && (typ == Float32 || typ == Float64)
rmw = (rmwop === :add ? "fadd" : "fsub")
@eval $fn(x::$container{$typ}, v::$typ) =
llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 acq_rel
ret $lt %rv
""", $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v)
end
end
end

# Provide atomic floating-point operations via atomic_cas!
# Provide atomic floating-point operations via atomic_cas! for those who lack support by llvm
const opnames = Dict{Symbol, Symbol}(:+ => :add, :- => :sub)
for op in [:+, :-, :max, :min]
for op in [:+, :-, :max, :min], container in [Atomic, Ptr]
fT = (op == :+ || op == :-) ? Float16 : FloatTypes
opname = get(opnames, op, op)
@eval function $(Symbol("atomic_", opname, "!"))(var::Atomic{T}, val::T) where T<:FloatTypes
@eval function $(Symbol("atomic_", opname, "!"))(var::$container{T}, val::T) where T<:$fT
IT = inttype(T)
old = var[]
while true
Expand All @@ -440,6 +449,7 @@ for op in [:+, :-, :max, :min]
end
end


"""
Threads.atomic_fence()

Expand Down
22 changes: 22 additions & 0 deletions test/threads_exec.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,28 @@ let atomictypes = intersect((Int8, Int16, Int32, Int64, Int128,
end
end

#Test that atomic operations work (at all) on pointers
#Since code is shared with ops on Atomic{T}, there is no need to test atomicity separately.
let atomictypes = intersect((Int8, Int16, Int32, Int64, Int128,
UInt8, UInt16, UInt32, UInt64, UInt128,
Float16, Float32, Float64), Base.Threads.atomictypes)

for T in atomictypes
arr = [zero(T)]
ptr = pointer(arr)
GC.@preserve arr begin
@test T(0) == atomic_add!(ptr, T(1))
@test T(1) == atomic_xchg!(ptr, T(4))
@test T(4) == atomic_sub!(ptr, T(1))
@test T(3) == atomic_max!(ptr, T(7))
@test T(7) == atomic_min!(ptr, T(10))
@test T(7) == atomic_cas!(ptr, T(7), T(5))
@test T(5) == atomic_cas!(ptr, T(7), T(6))
@test arr[1] == T(5)
end
end
end

# Test atomic_cas! and atomic_xchg!
function test_atomic_cas!(var::Atomic{T}, range::StepRange{Int,Int}) where T
for i in range
Expand Down