Skip to content

Commit

Permalink
precompile: do not reanimate zombie external methods (#48475)
Browse files Browse the repository at this point in the history 
Backedges are only applicable to cache objects with max_world==Inf

Fix #48391
  • Loading branch information
@vtjnash
vtjnash authored Feb 4, 2023
1 parent c4fd8a4 commit b9398a3
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 88 deletions.
12 changes: 3 additions & 9 deletions src/staticdata.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2181,7 +2181,6 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new
// edges: [caller1, ext_targets_indexes1, ...] for worklist-owned methods calling external methods
assert(edges_map == NULL);

htable_new(&external_mis, 0); // we need external_mis until after `jl_collect_edges` finishes
// Save the inferred code from newly inferred, external methods
*new_specializations = queue_external_cis(newly_inferred);

Expand Down Expand Up @@ -2209,14 +2208,9 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new
*edges = jl_alloc_vec_any(0);
*method_roots_list = jl_alloc_vec_any(0);
// Collect the new method roots
htable_t methods_with_newspecs;
htable_new(&methods_with_newspecs, 0);
jl_collect_methods(&methods_with_newspecs, *new_specializations);
jl_collect_new_roots(*method_roots_list, &methods_with_newspecs, worklist_key);
htable_free(&methods_with_newspecs);
jl_collect_edges(*edges, *ext_targets);
}
htable_free(&external_mis);
jl_collect_new_roots(*method_roots_list, *new_specializations, worklist_key);
jl_collect_edges(*edges, *ext_targets, *new_specializations);
}
assert(edges_map == NULL); // jl_collect_edges clears this when done

JL_GC_POP();
Expand Down
84 changes: 40 additions & 44 deletions src/staticdata_utils.c
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
static htable_t new_code_instance_validate;
static htable_t external_mis;


// inverse of backedges graph (caller=>callees hash)
jl_array_t *edges_map JL_GLOBALLY_ROOTED = NULL; // rooted for the duration of our uses of this
Expand Down Expand Up @@ -108,11 +108,6 @@ JL_DLLEXPORT void jl_push_newly_inferred(jl_value_t* ci)
}


static int method_instance_in_queue(jl_method_instance_t *mi)
{
return ptrhash_get(&external_mis, mi) != HT_NOTFOUND;
}

// compute whether a type references something internal to worklist
// and thus could not have existed before deserialize
// and thus does not need delayed unique-ing
Expand Down Expand Up @@ -216,10 +211,10 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited,
return found;
}

// given the list of CodeInstances that were inferred during the
// build, select those that are (1) external, and (2) are inferred to be called
// from the worklist or explicitly added by a `precompile` statement.
// Also prepares for method_instance_in_queue queries.
// Given the list of CodeInstances that were inferred during the build, select
// those that are (1) external, (2) still valid, and (3) are inferred to be
// called from the worklist or explicitly added by a `precompile` statement.
// These will be preserved in the image.
static jl_array_t *queue_external_cis(jl_array_t *list)
{
if (list == NULL)
Expand All @@ -238,17 +233,12 @@ static jl_array_t *queue_external_cis(jl_array_t *list)
assert(jl_is_code_instance(ci));
jl_method_instance_t *mi = ci->def;
jl_method_t *m = mi->def.method;
if (jl_is_method(m)) {
if (jl_object_in_image((jl_value_t*)m->module)) {
if (ptrhash_get(&external_mis, mi) == HT_NOTFOUND) {
int found = has_backedge_to_worklist(mi, &visited, &stack);
assert(found == 0 || found == 1);
assert(stack.len == 0);
if (found == 1) {
ptrhash_put(&external_mis, mi, mi);
jl_array_ptr_1d_push(new_specializations, (jl_value_t*)ci);
}
}
if (jl_is_method(m) && jl_object_in_image((jl_value_t*)m->module)) {
int found = has_backedge_to_worklist(mi, &visited, &stack);
assert(found == 0 || found == 1);
assert(stack.len == 0);
if (found == 1 && ci->max_world == ~(size_t)0) {
jl_array_ptr_1d_push(new_specializations, (jl_value_t*)ci);
}
}
}
Expand All @@ -259,31 +249,25 @@ static jl_array_t *queue_external_cis(jl_array_t *list)
}

// New roots for external methods
static void jl_collect_methods(htable_t *mset, jl_array_t *new_specializations)
static void jl_collect_new_roots(jl_array_t *roots, jl_array_t *new_specializations, uint64_t key)
{
size_t i, l = new_specializations ? jl_array_len(new_specializations) : 0;
jl_value_t *v;
jl_method_t *m;
for (i = 0; i < l; i++) {
v = jl_array_ptr_ref(new_specializations, i);
assert(jl_is_code_instance(v));
m = ((jl_code_instance_t*)v)->def->def.method;
htable_t mset;
htable_new(&mset, 0);
size_t l = new_specializations ? jl_array_len(new_specializations) : 0;
for (size_t i = 0; i < l; i++) {
jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(new_specializations, i);
assert(jl_is_code_instance(ci));
jl_method_t *m = ci->def->def.method;
assert(jl_is_method(m));
ptrhash_put(mset, (void*)m, (void*)m);
ptrhash_put(&mset, (void*)m, (void*)m);
}
}

static void jl_collect_new_roots(jl_array_t *roots, const htable_t *mset, uint64_t key)
{
size_t i, sz = mset->size;
int nwithkey;
jl_method_t *m;
void *const *table = mset->table;
void *const *table = mset.table;
jl_array_t *newroots = NULL;
JL_GC_PUSH1(&newroots);
for (i = 0; i < sz; i += 2) {
for (size_t i = 0; i < mset.size; i += 2) {
if (table[i+1] != HT_NOTFOUND) {
m = (jl_method_t*)table[i];
jl_method_t *m = (jl_method_t*)table[i];
assert(jl_is_method(m));
nwithkey = nroots_with_key(m, key);
if (nwithkey) {
Expand All @@ -305,6 +289,7 @@ static void jl_collect_new_roots(jl_array_t *roots, const htable_t *mset, uint64
}
}
JL_GC_POP();
htable_free(&mset);
}

// Create the forward-edge map (caller => callees)
Expand Down Expand Up @@ -422,9 +407,18 @@ static void jl_record_edges(jl_method_instance_t *caller, arraylist_t *wq, jl_ar
// Extract `edges` and `ext_targets` from `edges_map`
// `edges` = [caller1, targets_indexes1, ...], the list of methods and their edges
// `ext_targets` is [invokesig1, callee1, matches1, ...], the edges for each target
static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets)
static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *external_cis)
{
size_t world = jl_atomic_load_acquire(&jl_world_counter);
htable_t external_mis;
htable_new(&external_mis, 0);
if (external_cis) {
for (size_t i = 0; i < jl_array_len(external_cis); i++) {
jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(external_cis, i);
jl_method_instance_t *mi = ci->def;
ptrhash_put(&external_mis, (void*)mi, (void*)mi);
}
}
arraylist_t wq;
arraylist_new(&wq, 0);
void **table = (void**)jl_array_data(edges_map); // edges_map is caller => callees
Expand All @@ -438,10 +432,11 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets)
continue;
assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method));
if (!jl_object_in_image((jl_value_t*)caller->def.method->module) ||
method_instance_in_queue(caller)) {
ptrhash_get(&external_mis, caller) != HT_NOTFOUND) {
jl_record_edges(caller, &wq, edges);
}
}
htable_free(&external_mis);
while (wq.len) {
jl_method_instance_t *caller = (jl_method_instance_t*)arraylist_pop(&wq);
jl_record_edges(caller, &wq, edges);
Expand Down Expand Up @@ -1061,6 +1056,7 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_a
htable_reset(&visited, l);
for (i = 0; i < l; i++) {
jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(ci_list, i);
assert(ci->max_world == 1 || ci->max_world == ~(size_t)0);
assert(ptrhash_get(&visited, (void*)ci->def) == HT_NOTFOUND); // check that we don't have multiple cis for same mi
ptrhash_put(&visited, (void*)ci->def, (void*)ci);
}
Expand Down Expand Up @@ -1121,7 +1117,7 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_a
assert(jl_is_code_instance(ci));
jl_code_instance_t *codeinst = (jl_code_instance_t*)ci;
remove_code_instance_from_validation(codeinst); // mark it as handled
assert(codeinst->min_world >= world && codeinst->inferred);
assert(codeinst->min_world == world && codeinst->inferred);
codeinst->max_world = ~(size_t)0;
if (jl_rettype_inferred(caller, world, ~(size_t)0) == jl_nothing) {
jl_mi_cache_insert(caller, codeinst);
Expand Down Expand Up @@ -1161,8 +1157,8 @@ static void validate_new_code_instances(void)
//assert(0 && "unexpected unprocessed CodeInstance found");
jl_code_instance_t *ci = (jl_code_instance_t*)new_code_instance_validate.table[i];
JL_GC_PROMISE_ROOTED(ci); // TODO: this needs a root (or restructuring to avoid it)
assert(ci->min_world >= world && ci->inferred);
ci->max_world = ~(size_t)0;
assert(ci->min_world == world && ci->inferred);
assert(ci->max_world == ~(size_t)0);
jl_method_instance_t *caller = ci->def;
if (jl_rettype_inferred(caller, world, ~(size_t)0) == jl_nothing) {
jl_mi_cache_insert(caller, ci);
Expand Down
79 changes: 44 additions & 35 deletions test/precompile.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1686,64 +1686,73 @@ end
precompile_test_harness("DynamicExpressions") do load_path
# https://github.com/JuliaLang/julia/pull/47184#issuecomment-1364716312
write(joinpath(load_path, "Float16MWE.jl"),
"""
module Float16MWE
struct Node{T}
val::T
end
doconvert(::Type{<:Node}, val) = convert(Float16, val)
precompile(Tuple{typeof(doconvert), Type{Node{Float16}}, Float64})
end # module Float16MWE
""")
"""
module Float16MWE
struct Node{T}
val::T
end
doconvert(::Type{<:Node}, val) = convert(Float16, val)
precompile(Tuple{typeof(doconvert), Type{Node{Float16}}, Float64})
end # module Float16MWE
""")
Base.compilecache(Base.PkgId("Float16MWE"))
(@eval (using Float16MWE))
Base.invokelatest() do
@test Float16MWE.doconvert(Float16MWE.Node{Float16}, -1.2) === Float16(-1.2)
end
@eval using Float16MWE
@test @invokelatest(Float16MWE.doconvert(Float16MWE.Node{Float16}, -1.2)) === Float16(-1.2)
end

precompile_test_harness("BadInvalidations") do load_path
write(joinpath(load_path, "BadInvalidations.jl"),
"""
module BadInvalidations
"""
module BadInvalidations
Base.Experimental.@compiler_options compile=min optimize=1
getval() = Base.a_method_to_overwrite_in_test()
getval()
end # module BadInvalidations
""")
end # module BadInvalidations
""")
Base.compilecache(Base.PkgId("BadInvalidations"))
(@eval Base a_method_to_overwrite_in_test() = inferencebarrier(2))
(@eval (using BadInvalidations))
Base.invokelatest() do
@test BadInvalidations.getval() === 2
end
@eval Base a_method_to_overwrite_in_test() = inferencebarrier(2)
@eval using BadInvalidations
@test Base.invokelatest(BadInvalidations.getval) === 2
end

# https://github.com/JuliaLang/julia/issues/48074
precompile_test_harness("WindowsCacheOverwrite") do load_path
# https://github.com/JuliaLang/julia/pull/47184#issuecomment-1364716312
write(joinpath(load_path, "WindowsCacheOverwrite.jl"),
"""
module WindowsCacheOverwrite
end # module
""")
"""
module WindowsCacheOverwrite
end # module
""")
ji, ofile = Base.compilecache(Base.PkgId("WindowsCacheOverwrite"))
(@eval (using WindowsCacheOverwrite))
@eval using WindowsCacheOverwrite

write(joinpath(load_path, "WindowsCacheOverwrite.jl"),
"""
module WindowsCacheOverwrite
f() = "something new"
end # module
""")
"""
module WindowsCacheOverwrite
f() = "something new"
end # module
""")

ji_2, ofile_2 = Base.compilecache(Base.PkgId("WindowsCacheOverwrite"))
@test ofile_2 == Base.ocachefile_from_cachefile(ji_2)
end

precompile_test_harness("Issue #48391") do load_path
write(joinpath(load_path, "I48391.jl"),
"""
module I48391
struct SurrealFinite <: Real end
precompile(Tuple{typeof(Base.isless), SurrealFinite, SurrealFinite})
Base.:(<)(x::SurrealFinite, y::SurrealFinite) = "good"
end
""")
ji, ofile = Base.compilecache(Base.PkgId("I48391"))
@eval using I48391
x = Base.invokelatest(I48391.SurrealFinite)
@test Base.invokelatest(isless, x, x) === "good"
@test_throws ErrorException isless(x, x)
end

empty!(Base.DEPOT_PATH)
append!(Base.DEPOT_PATH, original_depot_path)
empty!(Base.LOAD_PATH)
Expand Down

0 comments on commit b9398a3

Please sign in to comment.