Merge pull request #26849 from JuliaLang/jn/new-inbounds

 [NewOptimizer] implement boundscheck elision

base/compiler/compiler.jl

@@ -103,6 +103,12 @@ using .Sort
 
 inlining_enabled() = (JLOptions().can_inline == 1)
 coverage_enabled() = (JLOptions().code_coverage != 0)
+function inbounds_option()
+    opt_check_bounds = JLOptions().check_bounds
+    opt_check_bounds == 0 && return :default
+    opt_check_bounds == 1 && return :on
+    return :off
+end
 
 include("compiler/utilities.jl")
 include("compiler/validation.jl")

base/compiler/optimize.jl

@@ -218,6 +218,9 @@ const SLOT_USEDUNDEF    = 32 # slot has uses that might raise UndefVarError
 
 # const SLOT_CALLED      = 64
 
+const IR_FLAG_INBOUNDS = 0x01
+
+
 # known affect-free calls (also effect-free)
 const _PURE_BUILTINS = Any[tuple, svec, fieldtype, apply_type, ===, isa, typeof, UnionAll, nfields]
 

base/compiler/ssair/driver.jl

@@ -114,26 +114,44 @@ function just_construct_ssa(ci::CodeInfo, code::Vector{Any}, nargs::Int, linetab
         end
         idx += 1
     end
-    reindex_labels!(code)
+    reindex_labels!(code) # update labels changed above
+
+    inbounds_depth = 0 # Number of stacked inbounds
     meta = Any[]
     lines = fill(0, length(code))
+    flags = fill(0x00, length(code))
     let loc = RefValue(1)
         for i = 1:length(code)
             stmt = code[i]
-            stmt = normalize(stmt, meta, linetable, loc)
+            if isexpr(stmt, :inbounds)
+                arg1 = stmt.args[1]
+                if arg1 === true # push
+                    inbounds_depth += 1
+                elseif arg1 === false # clear
+                    inbounds_depth = 0
+                elseif inbounds_depth > 0 # pop
+                    inbounds_depth -= 1
+                end
+                stmt = nothing
+            else
+                stmt = normalize(stmt, meta, linetable, loc)
+            end
             code[i] = stmt
             if !(stmt === nothing)
                 lines[i] = loc[]
+                if inbounds_depth > 0
+                     flags[i] |= IR_FLAG_INBOUNDS
+                end
             end
         end
     end
-    code = strip_trailing_junk!(code, lines)
+    code = strip_trailing_junk!(code, lines, flags)
     cfg = compute_basic_blocks(code)
     defuse_insts = scan_slot_def_use(nargs, ci, code)
     @timeit "domtree 1" domtree = construct_domtree(cfg)
     ir = let code = Any[nothing for _ = 1:length(code)]
              argtypes = ci.slottypes[1:(nargs+1)]
-            IRCode(code, Any[], lines, cfg, argtypes, mod, meta)
+            IRCode(code, Any[], lines, flags, cfg, argtypes, mod, meta)
         end
     @timeit "construct_ssa" ir = construct_ssa!(ci, code, ir, domtree, defuse_insts, nargs)
     return ir

base/compiler/ssair/inlining2.jl

@@ -1,4 +1,20 @@
-const InliningTodo = Tuple{Int, Tuple{Bool, Bool, Bool, Int}, Method, Vector{Any}, Vector{LineInfoNode}, IRCode, Bool}
+struct InliningTodo
+    idx::Int # The statement to replace
+    # Properties of the call - these determine how arguments
+    # need to be rewritten.
+    isva::Bool
+    isinvoke::Bool
+    isapply::Bool
+    na::Int
+    method::Method  # The method being inlined
+    sparams::Vector{Any} # The static parameters we computed for this call site
+    # The LineTable and IR of the inlinee
+    linetable::Vector{LineInfoNode}
+    ir::IRCode
+    # If the function being inlined is a single basic block we can use a
+    # simpler inlining algorithm. This flag determines whether that's allowed
+    linear_inline_eligible::Bool
+end
 
 function ssa_inlining_pass!(ir::IRCode, linetable::Vector{LineInfoNode}, sv::OptimizationState)
     # Go through the function, perfoming simple ininlingin (e.g. replacing call by constants
@@ -19,16 +35,22 @@ function batch_inline!(todo::Vector{InliningTodo}, ir::IRCode, linetable::Vector
     bb_rename = zeros(Int, length(ir.cfg.blocks))
     split_targets = BitSet()
     merged_orig_blocks = BitSet()
-    for (idx, _a, _b, _c, _d, ir2, lie) in todo
+    boundscheck = inbounds_option()
+    if boundscheck === :default && sv.src.propagate_inbounds
+        boundscheck = :propagate
+    end
+    for item in todo
+        local idx, ir2, lie
         # A linear inline does not modify the CFG
-        lie && continue
+        item.linear_inline_eligible && continue
+        inlinee_cfg = item.ir.cfg
         # Figure out if we need to split the BB
         need_split_before = false
         need_split = true
-        block = block_for_inst(ir.cfg, idx)
+        block = block_for_inst(ir.cfg, item.idx)
         last_block_idx = last(ir.cfg.blocks[block].stmts)
 
-        if !isempty(ir2.cfg.blocks[1].preds)
+        if !isempty(inlinee_cfg.blocks[1].preds)
             need_split_before = true
         end
 
@@ -43,7 +65,7 @@ function batch_inline!(todo::Vector{InliningTodo}, ir::IRCode, linetable::Vector
             need_split = false
             post_bb_id = -ir[SSAValue(last_block_idx)].label
         else
-            post_bb_id = length(new_cfg_blocks) + length(ir2.cfg.blocks) + (need_split_before ? 1 : 0)
+            post_bb_id = length(new_cfg_blocks) + length(inlinee_cfg.blocks) + (need_split_before ? 1 : 0)
             need_split = true #!(idx == last_block_idx)
         end
 
@@ -57,21 +79,21 @@ function batch_inline!(todo::Vector{InliningTodo}, ir::IRCode, linetable::Vector
         orig_succs = copy(new_cfg_blocks[end].succs)
         empty!(new_cfg_blocks[end].succs)
         if need_split_before
-            bb_rename_range = (1:length(ir2.cfg.blocks)) .+ length(new_cfg_blocks)
+            bb_rename_range = (1:length(inlinee_cfg.blocks)) .+ length(new_cfg_blocks)
             push!(new_cfg_blocks[end].succs, length(new_cfg_blocks)+1)
-            append!(new_cfg_blocks, ir2.cfg.blocks)
+            append!(new_cfg_blocks, inlinee_cfg.blocks)
         else
             # Merge the last block that was already there with the first block we're adding
-            bb_rename_range = (1:length(ir2.cfg.blocks)) .+ (length(new_cfg_blocks) - 1)
-            append!(new_cfg_blocks[end].succs, ir2.cfg.blocks[1].succs)
-            append!(new_cfg_blocks, ir2.cfg.blocks[2:end])
+            bb_rename_range = (1:length(inlinee_cfg.blocks)) .+ (length(new_cfg_blocks) - 1)
+            append!(new_cfg_blocks[end].succs, inlinee_cfg.blocks[1].succs)
+            append!(new_cfg_blocks, inlinee_cfg.blocks[2:end])
         end
         if need_split
             push!(new_cfg_blocks, BasicBlock(ir.cfg.blocks[block].stmts,
                 Int[], orig_succs))
             push!(split_targets, length(new_cfg_blocks))
         end
-        new_block_range = (length(new_cfg_blocks)-length(ir2.cfg.blocks)+1):length(new_cfg_blocks)
+        new_block_range = (length(new_cfg_blocks)-length(inlinee_cfg.blocks)+1):length(new_cfg_blocks)
         push!(inserted_block_ranges, new_block_range)
 
         # Fixup the edges of the newely added blocks
@@ -96,8 +118,8 @@ function batch_inline!(todo::Vector{InliningTodo}, ir::IRCode, linetable::Vector
 
         for (old_block, new_block) in enumerate(bb_rename_range)
             if (length(new_cfg_blocks[new_block].succs) == 0)
-                terminator_idx = last(ir2.cfg.blocks[old_block].stmts)
-                terminator = ir2[SSAValue(terminator_idx)]
+                terminator_idx = last(inlinee_cfg.blocks[old_block].stmts)
+                terminator = item.ir[SSAValue(terminator_idx)]
                 if isa(terminator, ReturnNode) && isdefined(terminator, :val)
                     push!(new_cfg_blocks[new_block].succs, post_bb_id)
                     if need_split
@@ -142,19 +164,20 @@ function batch_inline!(todo::Vector{InliningTodo}, ir::IRCode, linetable::Vector
 
     let compact = IncrementalCompact(ir)
         compact.result_bbs = new_cfg_blocks
-        nnewnodes = length(compact.result) + (sum(todo) do (_a, _b, _c, _d, _e, ir2, _f)
-            return length(ir2.stmts) + length(ir2.new_nodes)
+        nnewnodes = length(compact.result) + (sum(todo) do item
+            return length(item.ir.stmts) + length(item.ir.new_nodes)
         end)
         resize!(compact, nnewnodes)
-        (inline_idx, (isva, isinvoke, isapply, na), method, spvals, inline_linetable, inline_ir, lie) = popfirst!(todo)
+        item = popfirst!(todo)
+        inline_idx = item.idx
         for (idx, stmt) in compact
             if compact.idx - 1 == inline_idx
                 # Ok, do the inlining here
-                inline_cfg = inline_ir.cfg
+                inline_cfg = item.ir.cfg
                 linetable_offset = length(linetable)
                 # Append the linetable of the inlined function to our line table
                 inlined_at = compact.result_lines[idx]
-                for entry in inline_linetable
+                for entry in item.linetable
                     push!(linetable, LineInfoNode(entry.mod, entry.method, entry.file, entry.line,
                         (entry.inlined_at > 0 ? entry.inlined_at + linetable_offset : inlined_at)))
                 end
@@ -174,23 +197,29 @@ function batch_inline!(todo::Vector{InliningTodo}, ir::IRCode, linetable::Vector
                     end
                 end
                 argexprs = rewrite_exprargs((node, typ)->insert_node_here!(compact, node, typ, compact.result_lines[idx]),
-                                            arg->compact_exprtype(compact, arg), isinvoke, isapply, argexprs)
-                if isva
-                    vararg = mk_tuplecall!(compact, argexprs[na:end], compact.result_lines[idx])
-                    argexprs = Any[argexprs[1:(na - 1)]..., vararg]
+                                            arg->compact_exprtype(compact, arg), item.isinvoke, item.isapply, argexprs)
+                if item.isva
+                    vararg = mk_tuplecall!(compact, argexprs[item.na:end], compact.result_lines[idx])
+                    argexprs = Any[argexprs[1:(item.na - 1)]..., vararg]
+                end
+                flag = compact.result_flags[idx]
+                boundscheck_idx = boundscheck
+                if boundscheck_idx === :default || boundscheck_idx === :propagate
+                    if (flag & IR_FLAG_INBOUNDS) != 0
+                        boundscheck_idx = :off
+                    end
                 end
                 # Special case inlining that maintains the current basic block if there's only one BB in the target
-                if lie
-                    terminator = inline_ir[SSAValue(last(inline_cfg.blocks[1].stmts))]
+                if item.linear_inline_eligible
+                    terminator = item.ir[SSAValue(last(inline_cfg.blocks[1].stmts))]
                     compact[idx] = nothing
-                    inline_compact = IncrementalCompact(compact, inline_ir, compact.result_idx)
+                    inline_compact = IncrementalCompact(compact, item.ir, compact.result_idx)
                     for (idx′, stmt′) in inline_compact
                         # This dance is done to maintain accurate usage counts in the
                         # face of rename_arguments! mutating in place - should figure out
                         # something better eventually.
                         inline_compact[idx′] = nothing
-                        stmt′ = ssa_substitute!(stmt′, argexprs, method.sig, spvals)
-                        compact.result_lines[idx′] += linetable_offset
+                        stmt′ = ssa_substitute!(idx′, stmt′, argexprs, item.method.sig, item.sparams, linetable_offset, boundscheck_idx, compact)
                         if isa(stmt′, ReturnNode)
                             isa(stmt′.val, SSAValue) && (compact.used_ssas[stmt′.val.id] += 1)
                             compact.ssa_rename[compact.idx-1] = stmt′.val
@@ -204,17 +233,16 @@ function batch_inline!(todo::Vector{InliningTodo}, ir::IRCode, linetable::Vector
                 else
                     bb_offset, post_bb_id = popfirst!(todo_bbs)
                     # This implements the need_split_before flag above
-                    need_split_before = !isempty(inline_ir.cfg.blocks[1].preds)
+                    need_split_before = !isempty(item.ir.cfg.blocks[1].preds)
                     if need_split_before
                         finish_current_bb!(compact)
                     end
                     pn = PhiNode()
                     compact[idx] = nothing
-                    inline_compact = IncrementalCompact(compact, inline_ir, compact.result_idx)
+                    inline_compact = IncrementalCompact(compact, item.ir, compact.result_idx)
                     for (idx′, stmt′) in inline_compact
                         inline_compact[idx′] = nothing
-                        stmt′ = ssa_substitute!(stmt′, argexprs, method.sig, spvals)
-                        compact.result_lines[idx′] += linetable_offset
+                        stmt′ = ssa_substitute!(idx′, stmt′, argexprs, item.method.sig, item.sparams, linetable_offset, boundscheck_idx, compact)
                         if isa(stmt′, ReturnNode)
                             if isdefined(stmt′, :val)
                                 push!(pn.edges, inline_compact.active_result_bb-1)
@@ -250,7 +278,8 @@ function batch_inline!(todo::Vector{InliningTodo}, ir::IRCode, linetable::Vector
                     refinish && finish_current_bb!(compact)
                 end
                 if !isempty(todo)
-                    (inline_idx, (isva, isinvoke, isapply, na), method, spvals, inline_linetable, inline_ir, lie) = popfirst!(todo)
+                    item = popfirst!(todo)
+                    inline_idx = item.idx
                 else
                     inline_idx = -1
                 end
@@ -568,7 +597,10 @@ function assemble_inline_todo!(ir::IRCode, linetable::Vector{LineInfoNode}, sv::
         end
         #verify_ir(ir2)
 
-        push!(todo, (idx, (isva, isinvoke, isapply, na), method, Any[methsp...], inline_linetable, ir2, linear_inline_eligible(ir2)))
+        push!(todo, InliningTodo(idx,
+            isva, isinvoke, isapply, na,
+            method, Any[methsp...],
+            inline_linetable, ir2, linear_inline_eligible(ir2)))
     end
     todo
 end
@@ -688,7 +720,16 @@ function late_inline_special_case!(ir::IRCode, idx::Int, stmt::Expr, atypes::Vec
     return false
 end
 
-function ssa_substitute!(@nospecialize(val), arg_replacements::Vector{Any}, @nospecialize(spsig), spvals::Vector{Any})
+function ssa_substitute!(idx::Int, @nospecialize(val), arg_replacements::Vector{Any},
+                         @nospecialize(spsig), spvals::Vector{Any},
+                         linetable_offset::Int, boundscheck::Symbol, compact::IncrementalCompact)
+    compact.result_flags[idx] &= ~IR_FLAG_INBOUNDS
+    compact.result_lines[idx] += linetable_offset
+    return ssa_substitute_op!(val, arg_replacements, spsig, spvals, boundscheck)
+end
+
+function ssa_substitute_op!(@nospecialize(val), arg_replacements::Vector{Any},
+                            @nospecialize(spsig), spvals::Vector{Any}, boundscheck::Symbol)
     if isa(val, Argument)
         return arg_replacements[val.n]
     end
@@ -710,21 +751,19 @@ function ssa_substitute!(@nospecialize(val), arg_replacements::Vector{Any}, @nos
                     e.args[3] = svec(argtuple...)
                 end
             end
-    #=
         elseif head === :boundscheck
-            if boundscheck === :propagate
-                return e
-            elseif boundscheck === :off
+            if boundscheck === :off # inbounds == true
                 return false
-            else
+            elseif boundscheck === :propagate
+                return e
+            else # on or default
                 return true
             end
-    =#
         end
     end
     urs = userefs(val)
     for op in urs
-        op[] = ssa_substitute!(op[], arg_replacements, spsig, spvals)
+        op[] = ssa_substitute_op!(op[], arg_replacements, spsig, spvals, boundscheck)
     end
     return urs[]
 end