diff --git a/NEWS.md b/NEWS.md index 421093b6e0681..25efd0eb665ee 100644 --- a/NEWS.md +++ b/NEWS.md @@ -35,6 +35,8 @@ Compiler/Runtime improvements `@nospecialize`-d call sites and avoiding excessive compilation. ([#44512]) * All the previous usages of `@pure`-macro in `Base` has been replaced with the preferred `Base.@assume_effects`-based annotations. ([#44776]) +* `invoke(f, invokesig, args...)` calls to a less-specific method than would normally be chosen + for `f(args...)` are no longer spuriously invalidated when loading package precompile files. ([#46010]) Command-line option changes --------------------------- diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 31e34506a9046..0f504b9337c3f 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -801,6 +801,13 @@ function collect_const_args(argtypes::Vector{Any}) end for i = 2:length(argtypes) ] end +function invoke_signature(invokesig::Vector{Any}) + unwrapconst(x) = isa(x, Const) ? x.val : x + + f, argtyps = unwrapconst(invokesig[2]), unwrapconst(invokesig[3]) + return Tuple{typeof(f), unwrap_unionall(argtyps).parameters...} +end + function concrete_eval_call(interp::AbstractInterpreter, @nospecialize(f), result::MethodCallResult, arginfo::ArgInfo, sv::InferenceState) concrete_eval_eligible(interp, f, result, arginfo, sv) || return nothing @@ -1631,7 +1638,7 @@ function abstract_invoke(interp::AbstractInterpreter, (; fargs, argtypes)::ArgIn ti = tienv[1]; env = tienv[2]::SimpleVector result = abstract_call_method(interp, method, ti, env, false, sv) (; rt, edge, effects) = result - edge !== nothing && add_backedge!(edge::MethodInstance, sv) + edge !== nothing && add_backedge!(edge::MethodInstance, sv, argtypes) match = MethodMatch(ti, env, method, argtype <: method.sig) res = nothing sig = match.spec_types diff --git a/base/compiler/inferencestate.jl b/base/compiler/inferencestate.jl index d81bdccb7fa1c..732cd86e34fcc 100644 --- a/base/compiler/inferencestate.jl +++ b/base/compiler/inferencestate.jl @@ -479,12 +479,15 @@ function add_cycle_backedge!(frame::InferenceState, caller::InferenceState, curr end # temporarily accumulate our edges to later add as backedges in the callee -function add_backedge!(li::MethodInstance, caller::InferenceState) +function add_backedge!(li::MethodInstance, caller::InferenceState, invokesig::Union{Nothing,Vector{Any}}=nothing) isa(caller.linfo.def, Method) || return # don't add backedges to toplevel exprs edges = caller.stmt_edges[caller.currpc] if edges === nothing edges = caller.stmt_edges[caller.currpc] = [] end + if invokesig !== nothing + push!(edges, invoke_signature(invokesig)) + end push!(edges, li) return nothing end diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index e9c37a3054352..f19fbd014a04e 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -62,6 +62,10 @@ intersect!(et::EdgeTracker, range::WorldRange) = et.valid_worlds[] = intersect(et.valid_worlds[], range) push!(et::EdgeTracker, mi::MethodInstance) = push!(et.edges, mi) +function add_edge!(et::EdgeTracker, @nospecialize(invokesig), mi::MethodInstance) + invokesig === nothing && return push!(et.edges, mi) + push!(et.edges, invokesig, mi) +end function push!(et::EdgeTracker, ci::CodeInstance) intersect!(et, WorldRange(min_world(li), max_world(li))) push!(et, ci.def) diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 44d9067679d52..a59d9e2ec41e2 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -29,7 +29,9 @@ pass to apply its own inlining policy decisions. struct DelayedInliningSpec match::Union{MethodMatch, InferenceResult} argtypes::Vector{Any} + invokesig # either nothing or a signature (signature is for an `invoke` call) end +DelayedInliningSpec(match, argtypes) = DelayedInliningSpec(match, argtypes, nothing) struct InliningTodo # The MethodInstance to be inlined @@ -37,11 +39,11 @@ struct InliningTodo spec::Union{ResolvedInliningSpec, DelayedInliningSpec} end -InliningTodo(mi::MethodInstance, match::MethodMatch, argtypes::Vector{Any}) = - InliningTodo(mi, DelayedInliningSpec(match, argtypes)) +InliningTodo(mi::MethodInstance, match::MethodMatch, argtypes::Vector{Any}, invokesig=nothing) = + InliningTodo(mi, DelayedInliningSpec(match, argtypes, invokesig)) -InliningTodo(result::InferenceResult, argtypes::Vector{Any}) = - InliningTodo(result.linfo, DelayedInliningSpec(result, argtypes)) +InliningTodo(result::InferenceResult, argtypes::Vector{Any}, invokesig=nothing) = + InliningTodo(result.linfo, DelayedInliningSpec(result, argtypes, invokesig)) struct ConstantCase val::Any @@ -810,7 +812,7 @@ end function resolve_todo(todo::InliningTodo, state::InliningState, flag::UInt8) mi = todo.mi - (; match, argtypes) = todo.spec::DelayedInliningSpec + (; match, argtypes, invokesig) = todo.spec::DelayedInliningSpec et = state.et #XXX: update_valid_age!(min_valid[1], max_valid[1], sv) @@ -818,7 +820,7 @@ function resolve_todo(todo::InliningTodo, state::InliningState, flag::UInt8) inferred_src = match.src if isa(inferred_src, ConstAPI) # use constant calling convention - et !== nothing && push!(et, mi) + et !== nothing && add_edge!(et, invokesig, mi) return ConstantCase(quoted(inferred_src.val)) else src = inferred_src # ::Union{Nothing,CodeInfo} for NativeInterpreter @@ -829,7 +831,7 @@ function resolve_todo(todo::InliningTodo, state::InliningState, flag::UInt8) if code isa CodeInstance if use_const_api(code) # in this case function can be inlined to a constant - et !== nothing && push!(et, mi) + et !== nothing && add_edge!(et, invokesig, mi) return ConstantCase(quoted(code.rettype_const)) else src = @atomic :monotonic code.inferred @@ -851,7 +853,7 @@ function resolve_todo(todo::InliningTodo, state::InliningState, flag::UInt8) src === nothing && return compileable_specialization(et, match, effects) - et !== nothing && push!(et, mi) + et !== nothing && add_edge!(et, invokesig, mi) return InliningTodo(mi, retrieve_ir_for_inlining(mi, src), effects) end @@ -873,7 +875,7 @@ function validate_sparams(sparams::SimpleVector) return true end -function analyze_method!(match::MethodMatch, argtypes::Vector{Any}, +function analyze_method!(match::MethodMatch, argtypes::Vector{Any}, invokesig, flag::UInt8, state::InliningState) method = match.method spec_types = match.spec_types @@ -905,7 +907,7 @@ function analyze_method!(match::MethodMatch, argtypes::Vector{Any}, mi = specialize_method(match; preexisting=true) # Union{Nothing, MethodInstance} isa(mi, MethodInstance) || return compileable_specialization(et, match, Effects()) - todo = InliningTodo(mi, match, argtypes) + todo = InliningTodo(mi, match, argtypes, invokesig) # If we don't have caches here, delay resolving this MethodInstance # until the batch inlining step (or an external post-processing pass) state.mi_cache === nothing && return todo @@ -1100,9 +1102,10 @@ function inline_invoke!( if isa(result, ConcreteResult) item = concrete_result_item(result, state) else + invokesig = invoke_signature(sig.argtypes) argtypes = invoke_rewrite(sig.argtypes) if isa(result, ConstPropResult) - (; mi) = item = InliningTodo(result.result, argtypes) + (; mi) = item = InliningTodo(result.result, argtypes, invokesig) validate_sparams(mi.sparam_vals) || return nothing if argtypes_to_type(argtypes) <: mi.def.sig state.mi_cache !== nothing && (item = resolve_todo(item, state, flag)) @@ -1110,7 +1113,7 @@ function inline_invoke!( return nothing end end - item = analyze_method!(match, argtypes, flag, state) + item = analyze_method!(match, argtypes, invokesig, flag, state) end handle_single_case!(ir, idx, stmt, item, todo, state.params, true) return nothing @@ -1328,7 +1331,7 @@ function handle_match!( # during abstract interpretation: for the purpose of inlining, we can just skip # processing this dispatch candidate _any(case->case.sig === spec_types, cases) && return true - item = analyze_method!(match, argtypes, flag, state) + item = analyze_method!(match, argtypes, nothing, flag, state) item === nothing && return false push!(cases, InliningCase(spec_types, item)) return true @@ -1475,7 +1478,7 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState) if isa(result, ConcreteResult) item = concrete_result_item(result, state) else - item = analyze_method!(info.match, sig.argtypes, flag, state) + item = analyze_method!(info.match, sig.argtypes, nothing, flag, state) end handle_single_case!(ir, idx, stmt, item, todo, state.params) end diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index 2c66083b9024b..46897769046ce 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -312,7 +312,9 @@ function CodeInstance( const_flags = 0x00 end end - relocatability = isa(inferred_result, Vector{UInt8}) ? inferred_result[end] : UInt8(0) + relocatability = isa(inferred_result, Vector{UInt8}) ? inferred_result[end] : + inferred_result === nothing ? UInt8(1) : UInt8(0) + # relocatability = isa(inferred_result, Vector{UInt8}) ? inferred_result[end] : UInt8(0) return CodeInstance(result.linfo, widenconst(result_type), rettype_const, inferred_result, const_flags, first(valid_worlds), last(valid_worlds), @@ -561,17 +563,12 @@ function store_backedges(frame::InferenceResult, edges::Vector{Any}) end function store_backedges(caller::MethodInstance, edges::Vector{Any}) - i = 1 - while i <= length(edges) - to = edges[i] + for (typ, to) in BackedgeIterator(edges) if isa(to, MethodInstance) - ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any), to, caller) - i += 1 + ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), to, typ, caller) else typeassert(to, Core.MethodTable) - typ = edges[i + 1] ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any, Any), to, typ, caller) - i += 2 end end end diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl index d793660195492..975dc3d1a9b5d 100644 --- a/base/compiler/utilities.jl +++ b/base/compiler/utilities.jl @@ -223,6 +223,65 @@ Check if `method` is declared as `Base.@constprop :none`. """ is_no_constprop(method::Union{Method,CodeInfo}) = method.constprop == 0x02 +############# +# backedges # +############# + +""" + BackedgeIterator(mi::MethodInstance) + BackedgeIterator(backedges::Vector{Any}) + +Return an iterator over a list of backedges, which may be extracted +from `mi`. Iteration returns `(sig, caller)` elements, which will be one of +the following: + +- `(nothing, caller::MethodInstance)`: a call made by ordinary inferrable dispatch +- `(invokesig, caller::MethodInstance)`: a call made by `invoke(f, invokesig, args...)` +- `(specsig, mt::MethodTable)`: an abstract call + +# Examples + +```julia +julia> callme(x) = x+1 +callme (generic function with 1 method) + +julia> callyou(x) = callme(x) +callyou (generic function with 1 method) + +julia> callyou(2.0) +3.0 + +julia> mi = first(which(callme, (Any,)).specializations) +MethodInstance for callme(::Float64) + +julia> @eval Core.Compiler for (sig, caller) in BackedgeIterator(Main.mi) + println(sig) + println(caller) + end +nothing +callyou(Float64) from callyou(Any) +``` +""" +struct BackedgeIterator + backedges::Vector{Any} +end + +const empty_backedge_iter = BackedgeIterator(Any[]) + +function BackedgeIterator(mi::MethodInstance) + isdefined(mi, :backedges) || return empty_backedge_iter + return BackedgeIterator(mi.backedges) +end + +function iterate(iter::BackedgeIterator, i::Int=1) + backedges = iter.backedges + i > length(backedges) && return nothing + item = backedges[i] + isa(item, MethodInstance) && return (nothing, item), i+1 # regular dispatch + isa(item, Core.MethodTable) && return (backedges[i+1], item), i+2 # abstract dispatch + return (item, backedges[i+1]::MethodInstance), i+2 # `invoke` calls +end + ######### # types # ######### diff --git a/src/dump.c b/src/dump.c index 27c10254eac51..a3a04f0f6d9bb 100644 --- a/src/dump.c +++ b/src/dump.c @@ -337,9 +337,10 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited) if (!mi->backedges) { return 0; } - size_t i, n = jl_array_len(mi->backedges); - for (i = 0; i < n; i++) { - jl_method_instance_t *be = (jl_method_instance_t*)jl_array_ptr_ref(mi->backedges, i); + size_t i = 0, n = jl_array_len(mi->backedges); + jl_method_instance_t *be; + while (i < n) { + i = get_next_backedge(mi->backedges, i, NULL, &be); if (has_backedge_to_worklist(be, visited)) { bp = ptrhash_bp(visited, mi); // re-acquire since rehashing might change the location *bp = (void*)((char*)HT_NOTFOUND + 2); // found @@ -369,7 +370,8 @@ static size_t queue_external_mis(jl_array_t *list) jl_code_instance_t *ci = mi->cache; int relocatable = 0; while (ci) { - relocatable |= ci->relocatability; + if (ci->max_world == ~(size_t)0) + relocatable |= ci->relocatability; ci = ci->next; } if (relocatable && ptrhash_get(&external_mis, mi) == HT_NOTFOUND) { @@ -947,12 +949,16 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li if (backedges) { // filter backedges to only contain pointers // to items that we will actually store (internal >= 2) - size_t ins, i, l = jl_array_len(backedges); - jl_method_instance_t **b_edges = (jl_method_instance_t**)jl_array_data(backedges); - for (ins = i = 0; i < l; i++) { - jl_method_instance_t *backedge = b_edges[i]; + size_t ins = 0, i = 0, l = jl_array_len(backedges); + jl_value_t **b_edges = (jl_value_t**)jl_array_data(backedges); + jl_value_t *invokeTypes; + jl_method_instance_t *backedge; + while (i < l) { + i = get_next_backedge(backedges, i, &invokeTypes, &backedge); if (module_in_worklist(backedge->def.method->module) || method_instance_in_queue(backedge)) { - b_edges[ins++] = backedge; + if (invokeTypes) + b_edges[ins++] = invokeTypes; + b_edges[ins++] = (jl_value_t*)backedge; } } if (ins != l) @@ -1168,7 +1174,9 @@ static void jl_collect_missing_backedges_to_mod(jl_methtable_t *mt) jl_array_t **edges = (jl_array_t**)ptrhash_bp(&edges_map, (void*)caller); if (*edges == HT_NOTFOUND) *edges = jl_alloc_vec_any(0); - jl_array_ptr_1d_push(*edges, missing_callee); + // To stay synchronized with the format from MethodInstances (specifically for `invoke`d calls), + // we have to push a pair of values. But in this case the callee is unknown, so we leave it NULL. + push_backedge(*edges, missing_callee, NULL); } } } @@ -1178,13 +1186,15 @@ static void collect_backedges(jl_method_instance_t *callee) JL_GC_DISABLED { jl_array_t *backedges = callee->backedges; if (backedges) { - size_t i, l = jl_array_len(backedges); - for (i = 0; i < l; i++) { - jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(backedges, i); + size_t i = 0, l = jl_array_len(backedges); + jl_value_t *invokeTypes; + jl_method_instance_t *caller; + while (i < l) { + i = get_next_backedge(backedges, i, &invokeTypes, &caller); jl_array_t **edges = (jl_array_t**)ptrhash_bp(&edges_map, caller); if (*edges == HT_NOTFOUND) *edges = jl_alloc_vec_any(0); - jl_array_ptr_1d_push(*edges, (jl_value_t*)callee); + push_backedge(*edges, invokeTypes, callee); } } } @@ -1268,6 +1278,15 @@ static void jl_collect_extext_methods_from_mod(jl_array_t *s, jl_module_t *m) JL } } +static void register_backedge(htable_t *all_callees, jl_value_t *invokeTypes, jl_value_t *c) +{ + if (invokeTypes) + ptrhash_put(all_callees, invokeTypes, c); + else + ptrhash_put(all_callees, c, c); + +} + // flatten the backedge map reachable from caller into callees static void jl_collect_backedges_to(jl_method_instance_t *caller, htable_t *all_callees) JL_GC_DISABLED { @@ -1278,11 +1297,13 @@ static void jl_collect_backedges_to(jl_method_instance_t *caller, htable_t *all_ *callees = *pcallees; assert(callees != HT_NOTFOUND); *pcallees = (jl_array_t*) HT_NOTFOUND; - size_t i, l = jl_array_len(callees); - for (i = 0; i < l; i++) { - jl_value_t *c = jl_array_ptr_ref(callees, i); - ptrhash_put(all_callees, c, c); - if (jl_is_method_instance(c)) { + size_t i = 0, l = jl_array_len(callees); + jl_method_instance_t *c; + jl_value_t *invokeTypes; + while (i < l) { + i = get_next_backedge(callees, i, &invokeTypes, &c); + register_backedge(all_callees, invokeTypes, (jl_value_t*)c); + if (c && jl_is_method_instance(c)) { jl_collect_backedges_to((jl_method_instance_t*)c, all_callees); } } @@ -1297,6 +1318,8 @@ static void jl_collect_backedges( /* edges */ jl_array_t *s, /* ext_targets */ j htable_t all_callees; // MIs called by worklist methods (eff. Set{MethodInstance}) htable_new(&all_targets, 0); htable_new(&all_callees, 0); + jl_value_t *invokeTypes; + jl_method_instance_t *c; size_t i; void **table = edges_map.table; // edges is caller => callees size_t table_size = edges_map.size; @@ -1309,10 +1332,10 @@ static void jl_collect_backedges( /* edges */ jl_array_t *s, /* ext_targets */ j continue; assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method)); if (module_in_worklist(caller->def.method->module) || method_instance_in_queue(caller)) { - size_t i, l = jl_array_len(callees); - for (i = 0; i < l; i++) { - jl_value_t *c = jl_array_ptr_ref(callees, i); - ptrhash_put(&all_callees, c, c); + size_t i = 0, l = jl_array_len(callees); + while (i < l) { + i = get_next_backedge(callees, i, &invokeTypes, &c); + register_backedge(&all_callees, invokeTypes, (jl_value_t*)c); if (jl_is_method_instance(c)) { jl_collect_backedges_to((jl_method_instance_t*)c, &all_callees); } @@ -1326,10 +1349,9 @@ static void jl_collect_backedges( /* edges */ jl_array_t *s, /* ext_targets */ j jl_value_t *callee = (jl_value_t*)pc[j]; void *target = ptrhash_get(&all_targets, (void*)callee); if (target == HT_NOTFOUND) { - jl_method_instance_t *callee_mi = (jl_method_instance_t*)callee; jl_value_t *sig; if (jl_is_method_instance(callee)) { - sig = callee_mi->specTypes; + sig = ((jl_method_instance_t*)callee)->specTypes; } else { sig = callee; @@ -2294,7 +2316,7 @@ void remove_code_instance_from_validation(jl_code_instance_t *codeinst) ptrhash_remove(&new_code_instance_validate, codeinst); } -static void jl_insert_method_instances(jl_array_t *list) +static void jl_insert_method_instances(jl_array_t *list) JL_GC_DISABLED { size_t i, l = jl_array_len(list); // Validate the MethodInstances @@ -2303,29 +2325,74 @@ static void jl_insert_method_instances(jl_array_t *list) size_t world = jl_atomic_load_acquire(&jl_world_counter); for (i = 0; i < l; i++) { jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(list, i); + int valid = 1; assert(jl_is_method_instance(mi)); if (jl_is_method(mi->def.method)) { - // Is this still the method we'd be calling? - jl_methtable_t *mt = jl_method_table_for(mi->specTypes); - struct jl_typemap_assoc search = {(jl_value_t*)mi->specTypes, world, NULL, 0, ~(size_t)0}; - jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(mt->defs, &search, /*offs*/0, /*subtype*/1); - if (entry) { - jl_value_t *mworld = entry->func.value; - if (jl_is_method(mworld) && mi->def.method != (jl_method_t*)mworld && jl_type_morespecific(((jl_method_t*)mworld)->sig, mi->def.method->sig)) { - jl_array_uint8_set(valids, i, 0); - invalidate_backedges(&remove_code_instance_from_validation, mi, world, "jl_insert_method_instance"); - // The codeinst of this mi haven't yet been removed - jl_code_instance_t *codeinst = mi->cache; - while (codeinst) { - remove_code_instance_from_validation(codeinst); - codeinst = codeinst->next; - } - if (_jl_debug_method_invalidation) { - jl_array_ptr_1d_push(_jl_debug_method_invalidation, mworld); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, jl_cstr_to_string("jl_method_table_insert")); // GC disabled + jl_method_t *m = mi->def.method; + if (m->deleted_world != ~(size_t)0) { + // The method we depended on has been deleted, invalidate + valid = 0; + } else { + // Is this still the method we'd be calling? + jl_methtable_t *mt = jl_method_table_for(mi->specTypes); + struct jl_typemap_assoc search = {(jl_value_t*)mi->specTypes, world, NULL, 0, ~(size_t)0}; + jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(mt->defs, &search, /*offs*/0, /*subtype*/1); + if (entry) { + jl_value_t *mworld = entry->func.value; + if (jl_is_method(mworld) && mi->def.method != (jl_method_t*)mworld && jl_type_morespecific(((jl_method_t*)mworld)->sig, mi->def.method->sig)) { + // There's still a chance this is valid, if any caller made this via `invoke` and the invoke-signature is still valid + assert(mi->backedges); // should not be NULL if it's on `list` + jl_value_t *invokeTypes; + jl_method_instance_t *caller; + size_t jins = 0, j0, j = 0, nbe = jl_array_len(mi->backedges); + while (j < nbe) { + j0 = j; + j = get_next_backedge(mi->backedges, j, &invokeTypes, &caller); + if (invokeTypes) { + struct jl_typemap_assoc search = {invokeTypes, world, NULL, 0, ~(size_t)0}; + entry = jl_typemap_assoc_by_type(mt->defs, &search, /*offs*/0, /*subtype*/1); + if (entry) { + jl_value_t *imworld = entry->func.value; + if (jl_is_method(imworld) && mi->def.method == (jl_method_t*)imworld) { + // this one is OK + // in case we deleted some earlier ones, move this earlier + for (; j0 < j; jins++, j0++) { + jl_array_ptr_set(mi->backedges, jins, jl_array_ptr_ref(mi->backedges, j0)); + } + continue; + } + } + } + invalidate_backedges(&remove_code_instance_from_validation, caller, world, "jl_insert_method_instance"); + // The codeinst of this mi haven't yet been removed + jl_code_instance_t *codeinst = caller->cache; + while (codeinst) { + remove_code_instance_from_validation(codeinst); + codeinst = codeinst->next; + } + } + jl_array_del_end(mi->backedges, j - jins); + if (jins == 0) { + m = (jl_method_t*)mworld; + valid = 0; + } } } } + if (!valid) { + // None of the callers were valid, so invalidate `mi` too + jl_array_uint8_set(valids, i, 0); + invalidate_backedges(&remove_code_instance_from_validation, mi, world, "jl_insert_method_instance"); + jl_code_instance_t *codeinst = mi->cache; + while (codeinst) { + remove_code_instance_from_validation(codeinst); + codeinst = codeinst->next; + } + if (_jl_debug_method_invalidation) { + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)m); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, jl_cstr_to_string("jl_method_table_insert")); // GC disabled + } + } } } // While it's tempting to just remove the invalidated MIs altogether, @@ -2342,36 +2409,42 @@ static void jl_insert_method_instances(jl_array_t *list) if (milive != mi) { // A previously-loaded module compiled this method, so the one we deserialized will be dropped. // But make sure the backedges are copied over. + jl_value_t *invokeTypes; + jl_method_instance_t *be, *belive; if (mi->backedges) { if (!milive->backedges) { // Copy all the backedges (after looking up the live ones) - size_t j, n = jl_array_len(mi->backedges); + size_t j = 0, jlive = 0, n = jl_array_len(mi->backedges); milive->backedges = jl_alloc_vec_any(n); jl_gc_wb(milive, milive->backedges); - for (j = 0; j < n; j++) { - jl_method_instance_t *be = (jl_method_instance_t*)jl_array_ptr_ref(mi->backedges, j); - jl_method_instance_t *belive = (jl_method_instance_t*)ptrhash_get(&uniquing_table, be); + while (j < n) { + j = get_next_backedge(mi->backedges, j, &invokeTypes, &be); + belive = (jl_method_instance_t*)ptrhash_get(&uniquing_table, be); if (belive == HT_NOTFOUND) belive = be; - jl_array_ptr_set(milive->backedges, j, belive); + jlive = set_next_backedge(milive->backedges, jlive, invokeTypes, belive); } } else { // Copy the missing backedges (this is an O(N^2) algorithm, but many methods have few MethodInstances) - size_t j, k, n = jl_array_len(mi->backedges), nlive = jl_array_len(milive->backedges); - for (j = 0; j < n; j++) { - jl_method_instance_t *be = (jl_method_instance_t*)jl_array_ptr_ref(mi->backedges, j); - jl_method_instance_t *belive = (jl_method_instance_t*)ptrhash_get(&uniquing_table, be); + size_t j = 0, k, n = jl_array_len(mi->backedges), nlive = jl_array_len(milive->backedges); + jl_value_t *invokeTypes2; + jl_method_instance_t *belive2; + while (j < n) { + j = get_next_backedge(mi->backedges, j, &invokeTypes, &be); + belive = (jl_method_instance_t*)ptrhash_get(&uniquing_table, be); if (belive == HT_NOTFOUND) belive = be; int found = 0; - for (k = 0; k < nlive; k++) { - if (belive == (jl_method_instance_t*)jl_array_ptr_ref(milive->backedges, k)) { + k = 0; + while (k < nlive) { + k = get_next_backedge(milive->backedges, k, &invokeTypes2, &belive2); + if (belive == belive2 && invokeTypes == invokeTypes2) { found = 1; break; } } if (!found) - jl_array_ptr_1d_push(milive->backedges, (jl_value_t*)belive); + push_backedge(milive->backedges, invokeTypes, belive); } } } @@ -2482,7 +2555,7 @@ static void jl_insert_backedges(jl_array_t *list, jl_array_t *targets) int32_t idx = idxs[j]; jl_value_t *callee = jl_array_ptr_ref(targets, idx * 2); if (jl_is_method_instance(callee)) { - jl_method_instance_add_backedge((jl_method_instance_t*)callee, caller); + jl_method_instance_add_backedge((jl_method_instance_t*)callee, NULL, caller); } else { jl_methtable_t *mt = jl_method_table_for(callee); diff --git a/src/gf.c b/src/gf.c index c9d91ec836a0b..6b2d9362198f7 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1445,18 +1445,23 @@ static void invalidate_method_instance(void (*f)(jl_code_instance_t*), jl_method codeinst->max_world = max_world; } assert(codeinst->max_world <= max_world); + JL_GC_PUSH1(&codeinst); (*f)(codeinst); + JL_GC_POP(); codeinst = jl_atomic_load_relaxed(&codeinst->next); } // recurse to all backedges to update their valid range also jl_array_t *backedges = replaced->backedges; if (backedges) { + JL_GC_PUSH1(&backedges); replaced->backedges = NULL; - size_t i, l = jl_array_len(backedges); - for (i = 0; i < l; i++) { - jl_method_instance_t *replaced = (jl_method_instance_t*)jl_array_ptr_ref(backedges, i); + size_t i = 0, l = jl_array_len(backedges); + jl_method_instance_t *replaced; + while (i < l) { + i = get_next_backedge(backedges, i, NULL, &replaced); invalidate_method_instance(f, replaced, max_world, depth + 1); } + JL_GC_POP(); } JL_UNLOCK(&replaced->def.method->writelock); } @@ -1469,10 +1474,11 @@ void invalidate_backedges(void (*f)(jl_code_instance_t*), jl_method_instance_t * if (backedges) { // invalidate callers (if any) replaced_mi->backedges = NULL; - size_t i, l = jl_array_len(backedges); - jl_method_instance_t **replaced = (jl_method_instance_t**)jl_array_ptr_data(backedges); - for (i = 0; i < l; i++) { - invalidate_method_instance(f, replaced[i], max_world, 1); + size_t i = 0, l = jl_array_len(backedges); + jl_method_instance_t *replaced; + while (i < l) { + i = get_next_backedge(backedges, i, NULL, &replaced); + invalidate_method_instance(f, replaced, max_world, 1); } } JL_UNLOCK(&replaced_mi->def.method->writelock); @@ -1486,23 +1492,34 @@ void invalidate_backedges(void (*f)(jl_code_instance_t*), jl_method_instance_t * } // add a backedge from callee to caller -JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_method_instance_t *caller) +JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_value_t *invokesig, jl_method_instance_t *caller) { JL_LOCK(&callee->def.method->writelock); + if (invokesig == jl_nothing) + invokesig = NULL; // julia uses `nothing` but C uses NULL (#undef) if (!callee->backedges) { // lazy-init the backedges array - callee->backedges = jl_alloc_vec_any(1); + callee->backedges = jl_alloc_vec_any(0); jl_gc_wb(callee, callee->backedges); - jl_array_ptr_set(callee->backedges, 0, caller); + push_backedge(callee->backedges, invokesig, caller); } else { - size_t i, l = jl_array_len(callee->backedges); - for (i = 0; i < l; i++) { - if (jl_array_ptr_ref(callee->backedges, i) == (jl_value_t*)caller) + size_t i = 0, l = jl_array_len(callee->backedges); + int found = 0; + jl_value_t *invokeTypes; + jl_method_instance_t *mi; + while (i < l) { + i = get_next_backedge(callee->backedges, i, &invokeTypes, &mi); + // TODO: it would be better to canonicalize (how?) the Tuple-type so + // that we don't have to call `jl_egal` + if (mi == caller && ((invokesig == NULL && invokeTypes == NULL) || + (invokesig && invokeTypes && jl_egal(invokesig, invokeTypes)))) { + found = 1; break; + } } - if (i == l) { - jl_array_ptr_1d_push(callee->backedges, (jl_value_t*)caller); + if (!found) { + push_backedge(callee->backedges, invokesig, caller); } } JL_UNLOCK(&callee->def.method->writelock); diff --git a/src/jltypes.c b/src/jltypes.c index 8eb43076e46a5..2dc185db27c9b 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2532,7 +2532,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type, jl_simplevector_type, jl_any_type, - jl_any_type, + jl_array_any_type, jl_any_type, jl_any_type, jl_bool_type, diff --git a/src/julia.h b/src/julia.h index 30a3533156775..f780522081c3c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -362,7 +362,7 @@ struct _jl_method_instance_t { jl_value_t *specTypes; // argument types this was specialized for jl_svec_t *sparam_vals; // static parameter values, indexed by def.method->sparam_syms jl_value_t *uninferred; // cached uncompressed code, for generated functions, top-level thunks, or the interpreter - jl_array_t *backedges; // list of method-instances which contain a call into this method-instance + jl_array_t *backedges; // list of method-instances which call this method-instance; `invoke` records (invokesig, caller) pairs jl_array_t *callbacks; // list of callback functions to inform external caches about invalidations _Atomic(struct _jl_code_instance_t*) cache; uint8_t inInference; // flags to tell if inference is running on this object @@ -650,7 +650,7 @@ typedef struct _jl_methtable_t { intptr_t max_args; // max # of non-vararg arguments in a signature jl_value_t *kwsorter; // keyword argument sorter function jl_module_t *module; // used for incremental serialization to locate original binding - jl_array_t *backedges; + jl_array_t *backedges; // (sig, caller::MethodInstance) pairs jl_mutex_t writelock; uint8_t offs; // 0, or 1 to skip splitting typemap on first (function) argument uint8_t frozen; // whether this accepts adding new methods diff --git a/src/julia_internal.h b/src/julia_internal.h index 07ec5e8aedda2..8460e341e070d 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -604,6 +604,10 @@ JL_DLLEXPORT jl_code_info_t *jl_new_code_info_uninit(void); void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals, int binding_effects); +int get_next_backedge(jl_array_t *list, int i, jl_value_t** invokesig, jl_method_instance_t **caller) JL_NOTSAFEPOINT; +int set_next_backedge(jl_array_t *list, int i, jl_value_t *invokesig, jl_method_instance_t *caller); +void push_backedge(jl_array_t *list, jl_value_t *invokesig, jl_method_instance_t *caller); + JL_DLLEXPORT void jl_add_method_root(jl_method_t *m, jl_module_t *mod, jl_value_t* root); void jl_append_method_roots(jl_method_t *m, uint64_t modid, jl_array_t* roots); int get_root_reference(rle_reference *rr, jl_method_t *m, size_t i); @@ -949,7 +953,7 @@ JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *typ JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo( jl_method_t *m JL_PROPAGATES_ROOT, jl_value_t *type, jl_svec_t *sparams); jl_method_instance_t *jl_specializations_get_or_insert(jl_method_instance_t *mi_ins); -JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_method_instance_t *caller); +JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_value_t *invokesig, jl_method_instance_t *caller); JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_value_t *caller); uint32_t jl_module_next_counter(jl_module_t *m) JL_NOTSAFEPOINT; diff --git a/src/method.c b/src/method.c index 1abacfaa58e55..ce031cc1e5dc2 100644 --- a/src/method.c +++ b/src/method.c @@ -784,6 +784,49 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) return m; } +// backedges ------------------------------------------------------------------ + +// Use this in a `while` loop to iterate over the backedges in a MethodInstance. +// `*invokesig` will be NULL if the call was made by ordinary dispatch, otherwise +// it will be the signature supplied in an `invoke` call. +// If you don't need `invokesig`, you can set it to NULL on input. +// Initialize iteration with `i = 0`. Returns `i` for the next backedge to be extracted. +int get_next_backedge(jl_array_t *list, int i, jl_value_t** invokesig, jl_method_instance_t **caller) JL_NOTSAFEPOINT +{ + jl_value_t *item = jl_array_ptr_ref(list, i); + if (jl_is_method_instance(item)) { + // Not an `invoke` call, it's just the MethodInstance + if (invokesig != NULL) + *invokesig = NULL; + *caller = (jl_method_instance_t*)item; + return i + 1; + } + assert(jl_is_type(item)); + // An `invoke` call, it's a (sig, MethodInstance) pair + if (invokesig != NULL) + *invokesig = item; + *caller = (jl_method_instance_t*)jl_array_ptr_ref(list, i + 1); + if (*caller) + assert(jl_is_method_instance(*caller)); + return i + 2; +} + +int set_next_backedge(jl_array_t *list, int i, jl_value_t *invokesig, jl_method_instance_t *caller) +{ + if (invokesig) + jl_array_ptr_set(list, i++, invokesig); + jl_array_ptr_set(list, i++, caller); + return i; +} + +void push_backedge(jl_array_t *list, jl_value_t *invokesig, jl_method_instance_t *caller) +{ + if (invokesig) + jl_array_ptr_1d_push(list, invokesig); + jl_array_ptr_1d_push(list, (jl_value_t*)caller); + return; +} + // method definition ---------------------------------------------------------- jl_method_t *jl_make_opaque_closure_method(jl_module_t *module, jl_value_t *name, diff --git a/test/precompile.jl b/test/precompile.jl index ac2c63ff7af08..be4cbccf1172d 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -106,16 +106,18 @@ precompile_test_harness(false) do dir write(Foo2_file, """ module $Foo2_module - export override + export override, overridenc override(x::Integer) = 2 override(x::AbstractFloat) = Float64(override(1)) + overridenc(x::Integer) = rand()+1 + overridenc(x::AbstractFloat) = Float64(overridenc(1)) end """) write(Foo_file, """ module $Foo_module import $FooBase_module, $FooBase_module.typeA - import $Foo2_module: $Foo2_module, override + import $Foo2_module: $Foo2_module, override, overridenc import $FooBase_module.hash import Test module Inner @@ -221,6 +223,8 @@ precompile_test_harness(false) do dir g() = override(1.0) Test.@test g() === 2.0 # compile this + gnc() = overridenc(1.0) + Test.@test 1 < gnc() < 5 # compile this const abigfloat_f() = big"12.34" const abigfloat_x = big"43.21" @@ -257,6 +261,8 @@ precompile_test_harness(false) do dir Foo2 = Base.require(Main, Foo2_module) @eval $Foo2.override(::Int) = 'a' @eval $Foo2.override(::Float32) = 'b' + @eval $Foo2.overridenc(::Int) = rand() + 97.0 + @eval $Foo2.overridenc(::Float32) = rand() + 100.0 Foo = Base.require(Main, Foo_module) Base.invokelatest() do # use invokelatest to see the results of loading the compile @@ -265,9 +271,13 @@ precompile_test_harness(false) do dir # Issue #21307 @test Foo.g() === 97.0 + @test 96 < Foo.gnc() < 99 @test Foo.override(1.0e0) == Float64('a') @test Foo.override(1.0f0) == 'b' @test Foo.override(UInt(1)) == 2 + @test 96 < Foo.overridenc(1.0e0) < 99 + @test 99 < Foo.overridenc(1.0f0) < 102 + @test 0 < Foo.overridenc(UInt(1)) < 3 # Issue #15722 @test Foo.abigfloat_f()::BigFloat == big"12.34" @@ -873,6 +883,92 @@ precompile_test_harness("code caching") do dir @test hasvalid(mi, world) # was compiled with the new method end +precompile_test_harness("invoke") do dir + InvokeModule = :Invoke0x030e7e97c2365aad + CallerModule = :Caller0x030e7e97c2365aad + write(joinpath(dir, "$InvokeModule.jl"), + """ + module $InvokeModule + export f, g, h, fnc, gnc, hnc # nc variants do not infer to a Const + # f is for testing invoke that occurs within a dependency + f(x::Real) = 0 + f(x::Int) = x < 5 ? 1 : invoke(f, Tuple{Real}, x) + fnc(x::Real) = rand()-1 + fnc(x::Int) = x < 5 ? rand()+1 : invoke(fnc, Tuple{Real}, x) + # g is for testing invoke that occurs from a dependent + g(x::Real) = 0 + g(x::Int) = 1 + gnc(x::Real) = rand()-1 + gnc(x::Int) = rand()+1 + # h will be entirely superseded by a new method (full invalidation) + h(x::Real) = 0 + h(x::Int) = x < 5 ? 1 : invoke(h, Tuple{Integer}, x) + hnc(x::Real) = rand()-1 + hnc(x::Int) = x < 5 ? rand()+1 : invoke(hnc, Tuple{Integer}, x) + end + """) + write(joinpath(dir, "$CallerModule.jl"), + """ + module $CallerModule + using $InvokeModule + # involving external modules + callf(x) = f(x) + callg(x) = x < 5 ? g(x) : invoke(g, Tuple{Real}, x) + callh(x) = h(x) + callfnc(x) = fnc(x) + callgnc(x) = x < 5 ? gnc(x) : invoke(gnc, Tuple{Real}, x) + callhnc(x) = hnc(x) + + # Purely internal + internal(x::Real) = 0 + internal(x::Int) = x < 5 ? 1 : invoke(internal, Tuple{Real}, x) + internalnc(x::Real) = rand()-1 + internalnc(x::Int) = x < 5 ? rand()+1 : invoke(internalnc, Tuple{Real}, x) + + # force precompilation + begin + Base.Experimental.@force_compile + callf(3) + callg(3) + callh(3) + callfnc(3) + callgnc(3) + callhnc(3) + internal(3) + internalnc(3) + end + + # Now that we've precompiled, invalidate with a new method that overrides the `invoke` dispatch + $InvokeModule.h(x::Integer) = -1 + $InvokeModule.hnc(x::Integer) = rand() - 20 + end + """) + Base.compilecache(Base.PkgId(string(CallerModule))) + @eval using $CallerModule + M = getfield(@__MODULE__, CallerModule) + + function get_real_method(func) # return the method func(::Real) + for m in methods(func) + m.sig.parameters[end] === Real && return m + end + error("no ::Real method found for $func") + end + + for func in (M.f, M.g, M.internal, M.fnc, M.gnc, M.internalnc) + m = get_real_method(func) + mi = m.specializations[1] + @test length(mi.backedges) == 2 + @test mi.backedges[1] === Tuple{typeof(func), Real} + @test isa(mi.backedges[2], Core.MethodInstance) + @test mi.cache.max_world == typemax(mi.cache.max_world) + end + + m = get_real_method(M.h) + @test isempty(m.specializations) + m = get_real_method(M.hnc) + @test isempty(m.specializations) +end + # test --compiled-modules=no command line option precompile_test_harness("--compiled-modules=no") do dir Time_module = :Time4b3a94a1a081a8cb @@ -1069,14 +1165,22 @@ precompile_test_harness("delete_method") do dir """ module $A_module - export apc, anopc + export apc, anopc, apcnc, anopcnc + # Infer to a const apc(::Int, ::Int) = 1 apc(::Any, ::Any) = 2 anopc(::Int, ::Int) = 1 anopc(::Any, ::Any) = 2 + # Do not infer to a const + apcnc(::Int, ::Int) = rand() - 1 + apcnc(::Any, ::Any) = rand() + 1 + + anopcnc(::Int, ::Int) = rand() - 1 + anopcnc(::Any, ::Any) = rand() + 1 + end """) write(B_file, @@ -1087,19 +1191,26 @@ precompile_test_harness("delete_method") do dir bpc(x) = apc(x, x) bnopc(x) = anopc(x, x) + bpcnc(x) = apcnc(x, x) + bnopcnc(x) = anopcnc(x, x) precompile(bpc, (Int,)) precompile(bpc, (Float64,)) + precompile(bpcnc, (Int,)) + precompile(bpcnc, (Float64,)) end """) A = Base.require(Main, A_module) - for mths in (collect(methods(A.apc)), collect(methods(A.anopc))) - Base.delete_method(mths[1]) + for mths in (collect(methods(A.apc)), collect(methods(A.anopc)), collect(methods(A.apcnc)), collect(methods(A.anopcnc))) + idx = findfirst(m -> m.sig.parameters[end] === Int, mths) + Base.delete_method(mths[idx]) end B = Base.require(Main, B_module) - @test Base.invokelatest(B.bpc, 1) == Base.invokelatest(B.bpc, 1.0) == 2 - @test Base.invokelatest(B.bnopc, 1) == Base.invokelatest(B.bnopc, 1.0) == 2 + for f in (B.bpc, B.bnopc, B.bpcnc, B.bnopcnc) + @test Base.invokelatest(f, 1) > 1 + @test Base.invokelatest(f, 1.0) > 1 + end end precompile_test_harness("Issues #19030 and #25279") do load_path