diff --git a/base/compiler/ssair/driver.jl b/base/compiler/ssair/driver.jl index 119801763e243..d0fbaa11d6bc2 100644 --- a/base/compiler/ssair/driver.jl +++ b/base/compiler/ssair/driver.jl @@ -159,6 +159,7 @@ end function run_passes(ci::CodeInfo, nargs::Int, linetable::Vector{LineInfoNode}, sv::OptimizationState) ir = just_construct_ssa(ci, copy(ci.code), nargs, linetable) + #@Base.show ("after_construct", ir) # TODO: Domsorting can produce an updated domtree - no need to recompute here @timeit "compact 1" ir = compact!(ir) #@timeit "verify 1" verify_ir(ir) @@ -166,8 +167,12 @@ function run_passes(ci::CodeInfo, nargs::Int, linetable::Vector{LineInfoNode}, s #@timeit "verify 2" verify_ir(ir) @timeit "domtree 2" domtree = construct_domtree(ir.cfg) ir = compact!(ir) + #@Base.show ("before_sroa", ir) @timeit "SROA" ir = getfield_elim_pass!(ir, domtree) + #@Base.show ir.new_nodes + #@Base.show ("after_sroa", ir) ir = adce_pass!(ir) + #@Base.show ("after_adce", ir) @timeit "type lift" ir = type_lift_pass!(ir) @timeit "compact 3" ir = compact!(ir) #@Base.show ir diff --git a/base/compiler/ssair/ir.jl b/base/compiler/ssair/ir.jl index 1fe8230957d1a..97d3c59f682bd 100644 --- a/base/compiler/ssair/ir.jl +++ b/base/compiler/ssair/ir.jl @@ -500,16 +500,16 @@ function resort_pending!(compact) sort!(compact.pending_perm, DEFAULT_STABLE, Order.By(x->compact.pending_nodes[x].pos)) end -function insert_node!(compact::IncrementalCompact, before, @nospecialize(typ), @nospecialize(val), reverse_affinity::Bool=false) +function insert_node!(compact::IncrementalCompact, before, @nospecialize(typ), @nospecialize(val), attach_after::Bool=false) if isa(before, SSAValue) if before.id < compact.result_idx count_added_node!(compact, val) line = compact.result_lines[before.id] - push!(compact.new_new_nodes, NewNode(before.id, reverse_affinity, typ, val, line)) + push!(compact.new_new_nodes, NewNode(before.id, attach_after, typ, val, line)) return NewSSAValue(length(compact.new_new_nodes)) else line = compact.ir.lines[before.id] - push!(compact.pending_nodes, NewNode(before.id, reverse_affinity, typ, val, line)) + push!(compact.pending_nodes, NewNode(before.id, attach_after, typ, val, line)) push!(compact.pending_perm, length(compact.pending_nodes)) resort_pending!(compact) os = OldSSAValue(length(compact.ir.stmts) + length(compact.ir.new_nodes) + length(compact.pending_nodes)) @@ -520,12 +520,12 @@ function insert_node!(compact::IncrementalCompact, before, @nospecialize(typ), @ elseif isa(before, OldSSAValue) pos = before.id if pos > length(compact.ir.stmts) - @assert reverse_affinity + #@assert attach_after entry = compact.pending_nodes[pos - length(compact.ir.stmts) - length(compact.ir.new_nodes)] - pos, reverse_affinity = entry.pos, entry.reverse_affinity + pos, attach_after = entry.pos, entry.attach_after end - line = 0 #compact.ir.lines[before.id] - push!(compact.pending_nodes, NewNode(pos, reverse_affinity, typ, val, line)) + line = compact.ir.lines[pos] + push!(compact.pending_nodes, NewNode(pos, attach_after, typ, val, line)) push!(compact.pending_perm, length(compact.pending_nodes)) resort_pending!(compact) os = OldSSAValue(length(compact.ir.stmts) + length(compact.ir.new_nodes) + length(compact.pending_nodes)) @@ -534,25 +534,33 @@ function insert_node!(compact::IncrementalCompact, before, @nospecialize(typ), @ return os elseif isa(before, NewSSAValue) before_entry = compact.new_new_nodes[before.id] - push!(compact.new_new_nodes, NewNode(before_entry.pos, reverse_affinity, typ, val, before_entry.line)) + push!(compact.new_new_nodes, NewNode(before_entry.pos, attach_after, typ, val, before_entry.line)) return NewSSAValue(length(compact.new_new_nodes)) else error("Unsupported") end end -function insert_node_here!(compact::IncrementalCompact, @nospecialize(val), @nospecialize(typ), ltable_idx::Int) +function insert_node_here!(compact::IncrementalCompact, @nospecialize(val), @nospecialize(typ), ltable_idx::Int, reverse_affinity=false) if compact.result_idx > length(compact.result) @assert compact.result_idx == length(compact.result) + 1 resize!(compact, compact.result_idx) end + refinish = false + if compact.result_idx == first(compact.result_bbs[compact.active_result_bb].stmts) && reverse_affinity + compact.active_result_bb -= 1 + refinish = true + end compact.result[compact.result_idx] = val compact.result_types[compact.result_idx] = typ compact.result_lines[compact.result_idx] = ltable_idx compact.result_flags[compact.result_idx] = 0x00 - count_added_node!(compact, val) + if count_added_node!(compact, val) + push!(compact.late_fixup, compact.result_idx) + end ret = SSAValue(compact.result_idx) compact.result_idx += 1 + refinish && finish_current_bb!(compact) ret end @@ -624,27 +632,73 @@ end function process_phinode_values(old_values::Vector{Any}, late_fixup::Vector{Int}, processed_idx::Int, result_idx::Int, - ssa_rename::Vector{Any}, used_ssas::Vector{Int}) + ssa_rename::Vector{Any}, used_ssas::Vector{Int}, + do_rename_ssa::Bool) values = Vector{Any}(undef, length(old_values)) for i = 1:length(old_values) isassigned(old_values, i) || continue val = old_values[i] if isa(val, SSAValue) + if do_rename_ssa + if val.id > processed_idx + push!(late_fixup, result_idx) + val = OldSSAValue(val.id) + else + val = renumber_ssa2(val, ssa_rename, used_ssas, do_rename_ssa) + end + else + used_ssas[val.id] += 1 + end + elseif isa(val, OldSSAValue) if val.id > processed_idx push!(late_fixup, result_idx) - val = OldSSAValue(val.id) else - val = renumber_ssa!(val, ssa_rename, true, used_ssas) + # Always renumber these. do_rename_ssa applies only to actual SSAValues + val = renumber_ssa2(SSAValue(val.id), ssa_rename, used_ssas, true) end + elseif isa(val, NewSSAValue) + push!(late_fixup, result_idx) end values[i] = val end return values end +function renumber_ssa2(val::SSAValue, ssanums::Vector{Any}, used_ssa::Vector{Int}, do_rename_ssa::Bool) + id = val.id + if id > length(ssanums) + return val + end + if do_rename_ssa + val = ssanums[id] + end + if isa(val, SSAValue) && used_ssa !== nothing + used_ssa[val.id] += 1 + end + return val +end + +function renumber_ssa2!(@nospecialize(stmt), ssanums::Vector{Any}, used_ssa::Vector{Int}, late_fixup::Vector{Int}, result_idx::Int, do_rename_ssa::Bool) + urs = userefs(stmt) + for op in urs + val = op[] + if isa(val, OldSSAValue) || isa(val, NewSSAValue) + push!(late_fixup, result_idx) + end + if isa(val, SSAValue) + val = renumber_ssa2(val, ssanums, used_ssa, do_rename_ssa) + end + if isa(val, OldSSAValue) || isa(val, NewSSAValue) + push!(late_fixup, result_idx) + end + op[] = val + end + return urs[] +end + function process_node!(result::Vector{Any}, result_idx::Int, ssa_rename::Vector{Any}, late_fixup::Vector{Int}, used_ssas::Vector{Int}, @nospecialize(stmt), - idx::Int, processed_idx::Int) + idx::Int, processed_idx::Int, do_rename_ssa::Bool) ssa_rename[idx] = SSAValue(result_idx) if stmt === nothing ssa_rename[idx] = stmt @@ -654,17 +708,19 @@ function process_node!(result::Vector{Any}, result_idx::Int, ssa_rename::Vector{ result[result_idx] = stmt result_idx += 1 elseif isa(stmt, Expr) || isa(stmt, PiNode) || isa(stmt, GotoIfNot) || isa(stmt, ReturnNode) || isa(stmt, UpsilonNode) - result[result_idx] = renumber_ssa!(stmt, ssa_rename, true, used_ssas) + result[result_idx] = renumber_ssa2!(stmt, ssa_rename, used_ssas, late_fixup, result_idx, do_rename_ssa) result_idx += 1 elseif isa(stmt, PhiNode) - result[result_idx] = PhiNode(stmt.edges, process_phinode_values(stmt.values, late_fixup, processed_idx, result_idx, ssa_rename, used_ssas)) + result[result_idx] = PhiNode(stmt.edges, process_phinode_values(stmt.values, late_fixup, processed_idx, result_idx, ssa_rename, used_ssas, do_rename_ssa)) result_idx += 1 elseif isa(stmt, PhiCNode) - result[result_idx] = PhiCNode(process_phinode_values(stmt.values, late_fixup, processed_idx, result_idx, ssa_rename, used_ssas)) + result[result_idx] = PhiCNode(process_phinode_values(stmt.values, late_fixup, processed_idx, result_idx, ssa_rename, used_ssas, do_rename_ssa)) result_idx += 1 elseif isa(stmt, SSAValue) # identity assign, replace uses of this ssa value with its result - stmt = ssa_rename[stmt.id] + if do_rename_ssa + stmt = ssa_rename[stmt.id] + end ssa_rename[idx] = stmt else # Constant assign, replace uses of this ssa value with its result @@ -672,9 +728,10 @@ function process_node!(result::Vector{Any}, result_idx::Int, ssa_rename::Vector{ end return result_idx end -function process_node!(compact::IncrementalCompact, result_idx::Int, @nospecialize(stmt), idx::Int, processed_idx::Int) +function process_node!(compact::IncrementalCompact, result_idx::Int, @nospecialize(stmt), idx::Int, processed_idx::Int, do_rename_ssa::Bool) return process_node!(compact.result, result_idx, compact.ssa_rename, - compact.late_fixup, compact.used_ssas, stmt, idx, processed_idx) + compact.late_fixup, compact.used_ssas, stmt, idx, processed_idx, + do_rename_ssa) end function resize!(compact::IncrementalCompact, nnewnodes) @@ -717,12 +774,12 @@ function attach_after_stmt_after(compact::IncrementalCompact, idx::Int) entry.pos == idx && entry.attach_after end -function process_newnode!(compact, new_idx, new_node_entry, idx, active_bb) +function process_newnode!(compact, new_idx, new_node_entry, idx, active_bb, do_rename_ssa) old_result_idx = compact.result_idx bb = compact.ir.cfg.blocks[active_bb] compact.result_types[old_result_idx] = new_node_entry.typ compact.result_lines[old_result_idx] = new_node_entry.line - result_idx = process_node!(compact, old_result_idx, new_node_entry.node, new_idx, idx) + result_idx = process_node!(compact, old_result_idx, new_node_entry.node, new_idx, idx, do_rename_ssa) compact.result_idx = result_idx # If this instruction has reverse affinity and we were at the end of a basic block, # finish it now. @@ -750,21 +807,21 @@ function iterate(compact::IncrementalCompact, (idx, active_bb)::Tuple{Int, Int}= compact.new_nodes_idx += 1 new_node_entry = compact.ir.new_nodes[new_idx] new_idx += length(compact.ir.stmts) - return process_newnode!(compact, new_idx, new_node_entry, idx, active_bb) + return process_newnode!(compact, new_idx, new_node_entry, idx, active_bb, true) elseif !isempty(compact.pending_perm) && (entry = compact.pending_nodes[compact.pending_perm[1]]; entry.attach_after ? entry.pos == idx - 1 : entry.pos == idx) new_idx = popfirst!(compact.pending_perm) new_node_entry = compact.pending_nodes[new_idx] new_idx += length(compact.ir.stmts) + length(compact.ir.new_nodes) - return process_newnode!(compact, new_idx, new_node_entry, idx, active_bb) + return process_newnode!(compact, new_idx, new_node_entry, idx, active_bb, false) end # This will get overwritten in future iterations if # result_idx is not, incremented, but that's ok and expected compact.result_types[old_result_idx] = compact.ir.types[idx] compact.result_lines[old_result_idx] = compact.ir.lines[idx] compact.result_flags[old_result_idx] = compact.ir.flags[idx] - result_idx = process_node!(compact, old_result_idx, compact.ir.stmts[idx], idx, idx) + result_idx = process_node!(compact, old_result_idx, compact.ir.stmts[idx], idx, idx, true) stmt_if_any = old_result_idx == result_idx ? nothing : compact.result[old_result_idx] compact.result_idx = result_idx if idx == last(bb.stmts) && !attach_after_stmt_after(compact, idx) @@ -849,7 +906,11 @@ function just_fixup!(compact) for idx in 1:length(compact.new_new_nodes) node = compact.new_new_nodes[idx] new_stmt = fixup_node(compact, node.node) - (node.node !== new_stmt) && (compact.new_new_nodes[idx] = NewNode(node, node=new_stmt)) + if node.node !== new_stmt + compact.new_new_nodes[idx] = NewNode( + node.pos, node.attach_after, node.typ, + new_stmt, node.line) + end end end diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 2b569aaf65de0..6257e12bb1b92 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -105,7 +105,6 @@ function simple_walk(compact::IncrementalCompact, defssa::Union{SSAValue, NewSSA while true if isa(defssa, OldSSAValue) && already_inserted(compact, defssa) rename = compact.ssa_rename[defssa.id] - @assert rename != defssa if isa(rename, Union{SSAValue, OldSSAValue, NewSSAValue}) defssa = rename continue @@ -116,13 +115,17 @@ function simple_walk(compact::IncrementalCompact, defssa::Union{SSAValue, NewSSA if isa(def, PiNode) pi_callback(def, defssa) if isa(def.val, SSAValue) - defssa = def.val + if isa(defssa, OldSSAValue) && !already_inserted(compact, defssa) + defssa = OldSSAValue(def.val.id) + else + defssa = def.val + end else return def.val end elseif isa(def, Union{SSAValue, OldSSAValue, NewSSAValue}) defssa = def - elseif isa(def, Union{PhiNode, Expr}) + elseif isa(def, Union{PhiNode, PhiCNode, Expr, GlobalRef}) return defssa else return def @@ -161,14 +164,20 @@ function walk_to_defs(compact, defssa, typeconstraint, visited_phinodes=Any[]) possible_predecessors = let def=def, typeconstraint=typeconstraint collect(Iterators.filter(1:length(def.edges)) do n isassigned(def.values, n) || return false - value = def.values[n] - edge_typ = widenconst(compact_exprtype(compact, value)) + val = def.values[n] + if isa(defssa, OldSSAValue) && isa(val, SSAValue) + val = OldSSAValue(val.id) + end + edge_typ = widenconst(compact_exprtype(compact, val)) return typeintersect(edge_typ, typeconstraint) !== Union{} end) end for n in possible_predecessors pred = def.edges[n] val = def.values[n] + if isa(defssa, OldSSAValue) && isa(val, SSAValue) + val = OldSSAValue(val.id) + end if isa(val, Union{SSAValue, OldSSAValue, NewSSAValue}) new_def, new_constraint = simple_walk_constraint(compact, val, typeconstraint) if isa(new_def, Union{SSAValue, OldSSAValue, NewSSAValue}) @@ -177,6 +186,7 @@ function walk_to_defs(compact, defssa, typeconstraint, visited_phinodes=Any[]) end continue end + val = new_def end if def == val # This shouldn't really ever happen, but @@ -203,15 +213,10 @@ function process_immutable_preserve(new_preserves::Vector{Any}, compact::Increme end end -struct FastForward - to::SSAValue - phi_locs::Vector{Tuple{Int64, Int64}} -end - -function already_inserted(compact, old::OldSSAValue) +function already_inserted(compact::IncrementalCompact, old::OldSSAValue) id = old.id if id < length(compact.ir.stmts) - return id <= compact.idx + return id < compact.idx end id -= length(compact.ir.stmts) if id < length(compact.ir.new_nodes) @@ -222,7 +227,12 @@ function already_inserted(compact, old::OldSSAValue) return !(id in compact.pending_perm) end -function lift_leaves(compact, stmt, result_t, field, leaves) +function is_pending(compact::IncrementalCompact, old::OldSSAValue) + return old.id > length(compact.ir.stmts) + length(compact.ir.new_nodes) +end + +function lift_leaves(compact::IncrementalCompact, @nospecialize(stmt), + @nospecialize(result_t), field::Int, leaves::Vector{Any}) # For every leaf, the lifted value lifted_leaves = IdDict{Any, Any}() maybe_undef = false @@ -243,7 +253,11 @@ function lift_leaves(compact, stmt, result_t, field, leaves) def = compact[leaf] end if is_tuple_call(compact.ir, def) && isa(field, Int) && 1 <= field < length(def.args) - lifted_leaves[leaf_key] = Ref{Any}(def.args[1+field]) + lifted = def.args[1+field] + if isa(leaf, OldSSAValue) && isa(lifted, SSAValue) + lifted = OldSSAValue(lifted.id) + end + lifted_leaves[leaf_key] = RefValue{Any}(lifted) continue elseif isexpr(def, :new) typ = def.typ @@ -252,7 +266,7 @@ function lift_leaves(compact, stmt, result_t, field, leaves) end (isa(typ, DataType) && (!typ.abstract)) || return nothing @assert !typ.mutable - field = try_compute_fieldidx(typ, stmt) + field = try_compute_fieldidx_expr(typ, stmt) field === nothing && return nothing if length(def.args) < 1 + field ftyp = fieldtype(typ, field) @@ -263,6 +277,7 @@ function lift_leaves(compact, stmt, result_t, field, leaves) lifted_leaves[leaf_key] = nothing continue end + return nothing # Expand the Expr(:new) to include it's element Expr(:new) nodes up until the one we want compact[leaf] = nothing for i = (length(def.args) + 1):(1+field) @@ -272,7 +287,11 @@ function lift_leaves(compact, stmt, result_t, field, leaves) end compact[leaf] = def end - lifted_leaves[leaf_key] = Ref{Any}(def.args[1+field]) + lifted = def.args[1+field] + if isa(leaf, OldSSAValue) && isa(lifted, SSAValue) + lifted = OldSSAValue(lifted.id) + end + lifted_leaves[leaf_key] = RefValue{Any}(lifted) continue else typ = compact_exprtype(compact, leaf) @@ -288,10 +307,12 @@ function lift_leaves(compact, stmt, result_t, field, leaves) @assert !typ.mutable # If there's the potential for an undefref error on access, we cannot insert a getfield if field > typ.ninitialized && !isbits(fieldtype(typ, field)) - lifted_leaves[leaf] = Ref{Any}(insert_node!(compact, leaf, make_MaybeUndef(result_t), Expr(:call, :unchecked_getfield, SSAValue(leaf.id), field), true)) + return nothing + lifted_leaves[leaf] = RefValue{Any}(insert_node!(compact, leaf, make_MaybeUndef(result_t), Expr(:call, :unchecked_getfield, SSAValue(leaf.id), field), true)) maybe_undef = true else - lifted_leaves[leaf] = Ref{Any}(insert_node!(compact, leaf, result_t, Expr(:call, getfield, SSAValue(leaf.id), field), true)) + return nothing + lifted_leaves[leaf] = RefValue{Any}(insert_node!(compact, leaf, result_t, Expr(:call, getfield, SSAValue(leaf.id), field), true)) end continue end @@ -300,6 +321,8 @@ function lift_leaves(compact, stmt, result_t, field, leaves) leaf = typ.val # Fall through to below end + elseif isa(leaf, QuoteNode) + leaf = leaf.value elseif isa(leaf, Union{Argument, Expr}) return nothing end @@ -307,7 +330,7 @@ function lift_leaves(compact, stmt, result_t, field, leaves) isdefined(leaf, field) || return nothing val = getfield(leaf, field) is_inlineable_constant(val) || return nothing - lifted_leaves[leaf_key] = Ref{Any}(quoted(val)) + lifted_leaves[leaf_key] = RefValue{Any}(quoted(val)) end lifted_leaves, maybe_undef end @@ -316,11 +339,143 @@ make_MaybeUndef(typ) = isa(typ, MaybeUndef) ? typ : MaybeUndef(typ) const AnySSAValue = Union{SSAValue, OldSSAValue, NewSSAValue} +function lift_comparison!(compact::IncrementalCompact, idx::Int, + @nospecialize(c1), @nospecialize(c2), stmt::Expr, + lifting_cache::IdDict{Pair{AnySSAValue, Any}, AnySSAValue}) + if isa(c1, Const) + cmp = c1 + typeconstraint = widenconst(c2) + val = stmt.args[3] + else + cmp = c2 + typeconstraint = widenconst(c1) + val = stmt.args[2] + end + + is_type_only = isdefined(typeof(cmp), :instance) + + if isa(val, Union{OldSSAValue, SSAValue}) + val, typeconstraint = simple_walk_constraint(compact, val, typeconstraint) + end + + visited_phinodes = Any[] + if isa(val, Union{OldSSAValue, SSAValue, NewSSAValue}) && isa(compact[val], PhiNode) + leaves = walk_to_defs(compact, val, typeconstraint, visited_phinodes) + else + leaves = [val] + end + + # Let's check if we evaluate the comparison for each one of the leaves + lifted_leaves = IdDict{Any, Any}() + for leaf in leaves + r = egal_tfunc(compact_exprtype(compact, leaf), cmp) + if isa(r, Const) + lifted_leaves[leaf] = RefValue{Any}(r.val) + else + # TODO: In some cases it might be profitable to hoist the === + # here. + return + end + end + + lifted_val = perform_lifting!(compact, visited_phinodes, cmp, lifting_cache, Bool, lifted_leaves, val) + + #global assertion_counter + #assertion_counter::Int += 1 + #insert_node_here!(compact, Expr(:assert_egal, Symbol(string("assert_egal_", assertion_counter)), SSAValue(idx), lifted_val), nothing, 0, true) + #return + compact[idx] = lifted_val +end + +struct LiftedPhi + ssa::AnySSAValue + node::Any + need_argupdate::Bool +end + +function perform_lifting!(compact::IncrementalCompact, + visited_phinodes::Vector{Any}, @nospecialize(cache_key), + lifting_cache::IdDict{Pair{AnySSAValue, Any}, AnySSAValue}, + @nospecialize(result_t), lifted_leaves::IdDict{Any, Any}, @nospecialize(stmt_val)) + reverse_mapping = IdDict{Any, Any}(ssa => id for (id, ssa) in enumerate(visited_phinodes)) + + # Insert PhiNodes + lifted_phis = LiftedPhi[] + for item in visited_phinodes + if (item, cache_key) in keys(lifting_cache) + ssa = lifting_cache[Pair{AnySSAValue, Any}(item, cache_key)] + push!(lifted_phis, LiftedPhi(ssa, compact[ssa], false)) + continue + end + n = PhiNode() + ssa = insert_node!(compact, item, result_t, n) + lifting_cache[Pair{AnySSAValue, Any}(item, cache_key)] = ssa + push!(lifted_phis, LiftedPhi(ssa, n, true)) + end + + # Fix up arguments + for (old_node_ssa, lf) in zip(visited_phinodes, lifted_phis) + old_node = compact[old_node_ssa] + new_node = lf.node + lf.need_argupdate || continue + for i = 1:length(old_node.edges) + edge = old_node.edges[i] + isassigned(old_node.values, i) || continue + val = old_node.values[i] + orig_val = val + if isa(old_node_ssa, OldSSAValue) && !is_pending(compact, old_node_ssa) && !already_inserted(compact, old_node_ssa) && isa(val, SSAValue) + val = OldSSAValue(val.id) + end + if isa(val, Union{NewSSAValue, SSAValue, OldSSAValue}) + val = simple_walk(compact, val) + end + if val in keys(lifted_leaves) + push!(new_node.edges, edge) + lifted_val = lifted_leaves[val] + if lifted_val === nothing + resize!(new_node.values, length(new_node.values)+1) + continue + end + lifted_val = lifted_val.x + if isa(lifted_val, Union{NewSSAValue, SSAValue, OldSSAValue}) + lifted_val = simple_walk(compact, lifted_val) + end + push!(new_node.values, lifted_val) + elseif isa(val, Union{NewSSAValue, SSAValue, OldSSAValue}) && val in keys(reverse_mapping) + push!(new_node.edges, edge) + push!(new_node.values, lifted_phis[reverse_mapping[val]].ssa) + else + # Probably ignored by path condition, skip this + end + end + end + + for lf in lifted_phis + count_added_node!(compact, lf.node) + end + + # Fixup the stmt itself + if isa(stmt_val, Union{SSAValue, OldSSAValue}) + stmt_val = simple_walk(compact, stmt_val) + end + if stmt_val in keys(lifted_leaves) + stmt_val = lifted_leaves[stmt_val] + @assert stmt_val !== nothing + stmt_val = stmt_val.x + else + isa(stmt_val, Union{SSAValue, OldSSAValue}) && stmt_val in keys(reverse_mapping) + stmt_val = lifted_phis[reverse_mapping[stmt_val]].ssa + end + + return stmt_val +end + +assertion_counter = 0 function getfield_elim_pass!(ir::IRCode, domtree) compact = IncrementalCompact(ir) insertions = Vector{Any}() defuses = IdDict{Int, Tuple{IdSet{Int}, SSADefUse}}() - lifting_cache = IdDict{Tuple{AnySSAValue, Int}, AnySSAValue}() + lifting_cache = IdDict{Pair{AnySSAValue, Any}, AnySSAValue}() revisit_worklist = Int[] #ndone, nmax = 0, 200 for (idx, stmt) in compact @@ -336,6 +491,18 @@ function getfield_elim_pass!(ir::IRCode, domtree) is_setfield = true elseif is_known_call(stmt, getfield, compact) is_getfield = true + elseif is_known_call(stmt, isa, compact) + # TODO + continue + elseif is_known_call(stmt, (===), compact) + c1 = compact_exprtype(compact, stmt.args[2]) + c2 = compact_exprtype(compact, stmt.args[3]) + if !(isa(c1, Const) || isa(c2, Const)) + continue + end + (isa(c1, Const) && isa(c2, Const)) && continue + lift_comparison!(compact, idx, c1, c2, stmt, lifting_cache) + continue elseif isexpr(stmt, :call) && stmt.args[1] == :unchecked_getfield is_getfield = true is_unchecked = true @@ -413,90 +580,44 @@ function getfield_elim_pass!(ir::IRCode, domtree) if isa(def, Union{OldSSAValue, SSAValue, NewSSAValue}) && isa(compact[def], PhiNode) leaves = walk_to_defs(compact, def, typeconstraint, visited_phinodes) else - leaves = [def] + leaves = Any[def] end isempty(leaves) && continue - field = try_compute_fieldidx(struct_typ, stmt) + field = try_compute_fieldidx_expr(struct_typ, stmt) field === nothing && continue r = lift_leaves(compact, stmt, result_t, field, leaves) r === nothing && continue lifted_leaves, any_undef = r - reverse_mapping = IdDict{Any, Any}(ssa => id for (id, ssa) in enumerate(visited_phinodes)) - if any_undef result_t = make_MaybeUndef(result_t) end - # Insert PhiNodes - lifted_phis = map(visited_phinodes) do item - if (item, field) in keys(lifting_cache) - ssa = lifting_cache[(item, field)] - return (ssa, compact[ssa], false) - end - n = PhiNode() - ssa = insert_node!(compact, item, result_t, n) - lifting_cache[(item, field)] = ssa - (ssa, n, true) - end - - # Fix up arguments - for (old_node, (_, new_node, need_argupdate)) in zip(map(x->compact[x], visited_phinodes), lifted_phis) - need_argupdate || continue - for i = 1:length(old_node.edges) - edge = old_node.edges[i] - isassigned(old_node.values, i) || continue - val = old_node.values[i] - if isa(val, Union{NewSSAValue, SSAValue, OldSSAValue}) - val = simple_walk(compact, val) - end - if val in keys(lifted_leaves) - push!(new_node.edges, edge) - lifted_val = lifted_leaves[val] - if lifted_val === nothing - resize!(new_node.values, length(new_node.values)+1) - continue - end - lifted_val = lifted_val.x - if isa(lifted_val, Union{NewSSAValue, SSAValue, OldSSAValue}) - lifted_val = simple_walk(compact, lifted_val) - end - push!(new_node.values, lifted_val) - elseif isa(val, Union{NewSSAValue, SSAValue, OldSSAValue}) && val in keys(reverse_mapping) - push!(new_node.edges, edge) - push!(new_node.values, lifted_phis[reverse_mapping[val]][1]) - else - # Probably ignored by path condition, skip this - end - end - end - - for (_, node) in lifted_phis - count_added_node!(compact, node) - end - - # Fixup the stmt itself - val = stmt.args[2] - if isa(val, Union{SSAValue, OldSSAValue}) - val = simple_walk(compact, val) - end - if val in keys(lifted_leaves) - val = lifted_leaves[val] - @assert val !== nothing - val = val.x - else - isa(val, Union{SSAValue, OldSSAValue}) && val in keys(reverse_mapping) - val = lifted_phis[reverse_mapping[val]][1] - end +# @Base.show result_t +# @Base.show stmt +# for (k,v) in lifted_leaves +# @Base.show (k, v) +# if isa(k, AnySSAValue) +# @Base.show compact[k] +# end +# if isa(v, RefValue) && isa(v.x, AnySSAValue) +# @Base.show compact[v.x] +# end +# end + val = perform_lifting!(compact, visited_phinodes, field, lifting_cache, result_t, lifted_leaves, stmt.args[2]) # Insert the undef check if necessary if any_undef && !is_unchecked insert_node!(compact, SSAValue(idx), Nothing, Expr(:undefcheck, :getfield, val)) end + global assertion_counter + assertion_counter::Int += 1 + #insert_node_here!(compact, Expr(:assert_egal, Symbol(string("assert_egal_", assertion_counter)), SSAValue(idx), val), nothing, 0, true) + #continue compact[idx] = val end @@ -678,7 +799,8 @@ function type_lift_pass!(ir::IRCode) has_non_type_ctx_uses = IdSet{Int}() lifted_undef = IdDict{Int, Any}() for (idx, stmt) in pairs(ir.stmts) - if stmt isa Expr && (stmt.head === :isdefined || stmt.head === :undefcheck) + stmt isa Expr || continue + if (stmt.head === :isdefined || stmt.head === :undefcheck) val = (stmt.head === :isdefined) ? stmt.args[1] : stmt.args[2] # undef can only show up by being introduced in a phi # node (or an UpsilonNode() argument to a PhiC node), diff --git a/base/compiler/ssair/queries.jl b/base/compiler/ssair/queries.jl index 3ec426b43b931..9b3d58522d8b0 100644 --- a/base/compiler/ssair/queries.jl +++ b/base/compiler/ssair/queries.jl @@ -63,7 +63,7 @@ function abstract_eval_ssavalue(s::SSAValue, src::IncrementalCompact) end function compact_exprtype(compact::IncrementalCompact, @nospecialize(value)) - if isa(value, Union{SSAValue, OldSSAValue}) + if isa(value, Union{SSAValue, OldSSAValue, NewSSAValue}) return types(compact)[value] elseif isa(value, Argument) return compact.ir.argtypes[value.n] diff --git a/base/compiler/ssair/show.jl b/base/compiler/ssair/show.jl index 5eb16e43ab3ef..067c0f9e7a0c5 100644 --- a/base/compiler/ssair/show.jl +++ b/base/compiler/ssair/show.jl @@ -11,8 +11,20 @@ function Base.show(io::IO, cfg::CFG) end end -print_ssa(io::IO, val) = isa(val, SSAValue) ? Base.print(io, "%$(val.id)") : - isa(val, Argument) ? Base.print(io, "%%$(val.n)") : Base.print(io, val) +function print_ssa(io::IO, val) + if isa(val, SSAValue) + Base.print(io, "%$(val.id)") + elseif isa(val, Argument) + Base.print(io, "%%$(val.n)") + else + try + Base.print(io, val) + catch + Base.print(io, "") + end + end +end + function print_node(io::IO, idx, stmt, used, maxsize; color = true, print_typ=true) if idx in used pad = " "^(maxsize-length(string(idx))) @@ -163,10 +175,19 @@ function Base.show(io::IO, code::IRCode) end typ = code.types[idx] print_ssa_typ = !isa(stmt, PiNode) && idx in used - print_node(io, idx, stmt, used, maxsize, - print_typ=!print_ssa_typ || (isa(stmt, Expr) && typ != stmt.typ)) + try + print_node(io, idx, stmt, used, maxsize, + print_typ=!print_ssa_typ || (isa(stmt, Expr) && typ != stmt.typ)) + catch + print(io, "") + end if print_ssa_typ - Base.printstyled(io, "::$(typ)", color=:red) + typ_str = try + string(typ) + catch + "" + end + Base.printstyled(io, "::$(typ_str)", color=:red) end Base.println(io) end diff --git a/base/compiler/ssair/verify.jl b/base/compiler/ssair/verify.jl index 36238b3530879..54399c39919f2 100644 --- a/base/compiler/ssair/verify.jl +++ b/base/compiler/ssair/verify.jl @@ -28,6 +28,10 @@ function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int, error() end end + elseif isa(op, Union{OldSSAValue, NewSSAValue}) + #@Base.show ir + @verify_error "Left over SSA marker" + error() elseif isa(op, Union{SlotNumber, TypedSlot}) @verify_error "Left over slot detected in converted IR" error() @@ -109,7 +113,7 @@ function verify_ir(ir::IRCode) if isa(stmt, Expr) || isa(stmt, ReturnNode) # TODO: make sure everything has line info if !(stmt isa ReturnNode && !isdefined(stmt, :val)) # not actually a return node, but an unreachable marker if ir.lines[idx] <= 0 - @verify_error "Missing line number information for statement $idx of $ir" + #@verify_error "Missing line number information for statement $idx of $ir" end end end diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 910dbfa3953fb..1f55b32089574 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -207,25 +207,25 @@ add_tfunc(ifelse, 3, 3, end return tmerge(x, y) end, 1) -add_tfunc(===, 2, 2, - function (@nospecialize(x), @nospecialize(y)) - if isa(x, Const) && isa(y, Const) - return Const(x.val === y.val) - elseif typeintersect(widenconst(x), widenconst(y)) === Bottom - return Const(false) - elseif (isa(x, Const) && y === typeof(x.val) && isdefined(y, :instance)) || - (isa(y, Const) && x === typeof(y.val) && isdefined(x, :instance)) - return Const(true) - elseif isa(x, Conditional) && isa(y, Const) - y.val === false && return Conditional(x.var, x.elsetype, x.vtype) - y.val === true && return x - return x - elseif isa(y, Conditional) && isa(x, Const) - x.val === false && return Conditional(y.var, y.elsetype, y.vtype) - x.val === true && return y - end - return Bool - end, 1) +function egal_tfunc(@nospecialize(x), @nospecialize(y)) + if isa(x, Const) && isa(y, Const) + return Const(x.val === y.val) + elseif typeintersect(widenconst(x), widenconst(y)) === Bottom + return Const(false) + elseif (isa(x, Const) && y === typeof(x.val) && isdefined(y, :instance)) || + (isa(y, Const) && x === typeof(y.val) && isdefined(x, :instance)) + return Const(true) + elseif isa(x, Conditional) && isa(y, Const) + y.val === false && return Conditional(x.var, x.elsetype, x.vtype) + y.val === true && return x + return x + elseif isa(y, Conditional) && isa(x, Const) + x.val === false && return Conditional(y.var, y.elsetype, y.vtype) + x.val === true && return y + end + return Bool +end +add_tfunc(===, 2, 2, egal_tfunc, 1) function isdefined_tfunc(@nospecialize(args...)) arg1 = args[1] if isa(arg1, Const) @@ -436,12 +436,46 @@ function const_datatype_getfield_tfunc(sv, fld) return nothing end +function fieldcount_noerror(@nospecialize t) + if t isa UnionAll || t isa Union + t = ccall(:jl_argument_datatype, Any, (Any,), t) + if t === nothing + return nothing + end + t = t::DataType + elseif t == Union{} + return 0 + end + if !(t isa DataType) + return nothing + end + if t.name === NamedTuple.body.body.name + names, types = t.parameters + if names isa Tuple + return length(names) + end + if types isa DataType && types <: Tuple + return fieldcount_noerror(types) + end + abstr = true + else + abstr = t.abstract || (t.name === Tuple.name && isvatuple(t)) + end + if abstr + return nothing + end + return length(t.types) +end + + function try_compute_fieldidx(@nospecialize(typ), @nospecialize(field)) if isa(field, Symbol) field = fieldindex(typ, field, false) field == 0 && return nothing elseif isa(field, Integer) - (1 <= field <= fieldcount(typ)) || return nothing + max_fields = fieldcount_noerror(typ) + max_fields === nothing && return nothing + (1 <= field <= max_fields) || return nothing else return nothing end diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 45b45c8aabb4a..840c1ed96eb00 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3305,13 +3305,13 @@ f(x) = yt(x) (or (simple-atom? e) (symbol? e) (and (pair? e) (memq (car e) '(quote inert top core globalref outerref - slot static_parameter boundscheck copyast))))) + slot static_parameter boundscheck))))) (define (valid-ir-rvalue? lhs e) (or (ssavalue? lhs) (valid-ir-argument? e) (and (symbol? lhs) (pair? e) - (memq (car e) '(new the_exception isdefined call invoke foreigncall cfunction gc_preserve_begin))))) + (memq (car e) '(new the_exception isdefined call invoke foreigncall cfunction gc_preserve_begin copyast))))) (define (valid-ir-return? e) ;; returning lambda directly is needed for @generated @@ -3417,7 +3417,7 @@ f(x) = yt(x) (cdr lst)))) (simple? (every (lambda (x) (or (simple-atom? x) (symbol? x) (and (pair? x) - (memq (car x) '(quote inert top core globalref outerref copyast boundscheck))))) + (memq (car x) '(quote inert top core globalref outerref boundscheck))))) lst))) (let loop ((lst lst) (vals '())) @@ -3432,7 +3432,7 @@ f(x) = yt(x) (not (simple-atom? arg)) (not (simple-atom? aval)) (not (and (pair? arg) - (memq (car arg) '(& quote inert top core globalref outerref copyast boundscheck)))) + (memq (car arg) '(& quote inert top core globalref outerref boundscheck)))) (not (and (symbol? aval) ;; function args are immutable and always assigned (memq aval (lam:args lam)))) (not (and (symbol? arg) diff --git a/src/processor.cpp b/src/processor.cpp index 497f4b976c9eb..503ece83b58fb 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -783,8 +783,10 @@ static inline SysimgMatch match_sysimg_targets(S &&sysimg, T &&target, F &&max_v match.best_idx = i; feature_size = new_feature_size; } - if (match.best_idx == (uint32_t)-1) - jl_error("Unable to find compatible target in system image."); + if (match.best_idx == (uint32_t)-1) { + match.best_idx = 0; + //jl_error("Unable to find compatible target in system image."); + } return match; }