-
Notifications
You must be signed in to change notification settings - Fork 66
/
lower_load.jl
65 lines (61 loc) · 2.36 KB
/
lower_load.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
function pushvectorload!(q::Expr, op::Operation, var::Symbol, td::UnrollArgs, U::Int, W::Symbol, mask, vecnotunrolled::Bool)
@unpack u, unrolled = td
ptr = refname(op)
name, mo = name_memoffset(var, op, td, W, vecnotunrolled)
instrcall = Expr(:call, lv(:vload), ptr, mo)
if mask !== nothing && (vecnotunrolled || u == U - 1)
push!(instrcall.args, mask)
end
push!(q.args, Expr(:(=), name, instrcall))
end
function lower_load_scalar!(
q::Expr, op::Operation, vectorized::Symbol, W::Symbol, unrolled::Symbol, tiled::Symbol, U::Int,
suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned} = nothing
)
loopdeps = loopdependencies(op)
@assert vectorized ∉ loopdeps
var = variable_name(op, suffix)
ptr = refname(op)
isunrolled = unrolled ∈ loopdeps
U = isunrolled ? U : 1
for u ∈ 0:U-1
varname = varassignname(var, u, isunrolled)
td = UnrollArgs(u, unrolled, tiled, suffix)
push!(q.args, Expr(:(=), varname, Expr(:call, lv(:vload), ptr, mem_offset_u(op, td))))
end
nothing
end
function lower_load_vectorized!(
q::Expr, op::Operation, vectorized::Symbol, W::Symbol, unrolled::Symbol, tiled::Symbol, U::Int,
suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned} = nothing
)
loopdeps = loopdependencies(op)
@assert vectorized ∈ loopdeps
if unrolled ∈ loopdeps
umin = 0
U = U
else
umin = -1
U = 0
end
# Urange = unrolled ∈ loopdeps ? 0:U-1 : 0
var = variable_name(op, suffix)
vecnotunrolled = vectorized !== unrolled
for u ∈ umin:U-1
td = UnrollArgs(u, unrolled, tiled, suffix)
pushvectorload!(q, op, var, td, U, W, mask, vecnotunrolled)
end
nothing
end
# TODO: this code should be rewritten to be more "orthogonal", so that we're just combining separate pieces.
# Using sentinel values (eg, T = -1 for non tiling) in part to avoid recompilation.
function lower_load!(
q::Expr, op::Operation, vectorized::Symbol, W::Symbol, unrolled::Symbol, tiled::Symbol, U::Int,
suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned} = nothing
)
if vectorized ∈ loopdependencies(op)
lower_load_vectorized!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
else
lower_load_scalar!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask)
end
end