diff --git a/src/staticdata.c b/src/staticdata.c index 804193ff90229..94173c2a8a162 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -2181,7 +2181,6 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new // edges: [caller1, ext_targets_indexes1, ...] for worklist-owned methods calling external methods assert(edges_map == NULL); - htable_new(&external_mis, 0); // we need external_mis until after `jl_collect_edges` finishes // Save the inferred code from newly inferred, external methods *new_specializations = queue_external_cis(newly_inferred); @@ -2209,14 +2208,9 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new *edges = jl_alloc_vec_any(0); *method_roots_list = jl_alloc_vec_any(0); // Collect the new method roots - htable_t methods_with_newspecs; - htable_new(&methods_with_newspecs, 0); - jl_collect_methods(&methods_with_newspecs, *new_specializations); - jl_collect_new_roots(*method_roots_list, &methods_with_newspecs, worklist_key); - htable_free(&methods_with_newspecs); - jl_collect_edges(*edges, *ext_targets); - } - htable_free(&external_mis); + jl_collect_new_roots(*method_roots_list, *new_specializations, worklist_key); + jl_collect_edges(*edges, *ext_targets, *new_specializations); + } assert(edges_map == NULL); // jl_collect_edges clears this when done JL_GC_POP(); diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index fc109836a03c0..cb97db1083e32 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -1,5 +1,5 @@ static htable_t new_code_instance_validate; -static htable_t external_mis; + // inverse of backedges graph (caller=>callees hash) jl_array_t *edges_map JL_GLOBALLY_ROOTED = NULL; // rooted for the duration of our uses of this @@ -108,11 +108,6 @@ JL_DLLEXPORT void jl_push_newly_inferred(jl_value_t* ci) } -static int method_instance_in_queue(jl_method_instance_t *mi) -{ - return ptrhash_get(&external_mis, mi) != HT_NOTFOUND; -} - // compute whether a type references something internal to worklist // and thus could not have existed before deserialize // and thus does not need delayed unique-ing @@ -216,10 +211,10 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, return found; } -// given the list of CodeInstances that were inferred during the -// build, select those that are (1) external, and (2) are inferred to be called -// from the worklist or explicitly added by a `precompile` statement. -// Also prepares for method_instance_in_queue queries. +// Given the list of CodeInstances that were inferred during the build, select +// those that are (1) external, (2) still valid, and (3) are inferred to be +// called from the worklist or explicitly added by a `precompile` statement. +// These will be preserved in the image. static jl_array_t *queue_external_cis(jl_array_t *list) { if (list == NULL) @@ -238,17 +233,12 @@ static jl_array_t *queue_external_cis(jl_array_t *list) assert(jl_is_code_instance(ci)); jl_method_instance_t *mi = ci->def; jl_method_t *m = mi->def.method; - if (jl_is_method(m)) { - if (jl_object_in_image((jl_value_t*)m->module)) { - if (ptrhash_get(&external_mis, mi) == HT_NOTFOUND) { - int found = has_backedge_to_worklist(mi, &visited, &stack); - assert(found == 0 || found == 1); - assert(stack.len == 0); - if (found == 1) { - ptrhash_put(&external_mis, mi, mi); - jl_array_ptr_1d_push(new_specializations, (jl_value_t*)ci); - } - } + if (jl_is_method(m) && jl_object_in_image((jl_value_t*)m->module)) { + int found = has_backedge_to_worklist(mi, &visited, &stack); + assert(found == 0 || found == 1); + assert(stack.len == 0); + if (found == 1 && ci->max_world == ~(size_t)0) { + jl_array_ptr_1d_push(new_specializations, (jl_value_t*)ci); } } } @@ -259,31 +249,25 @@ static jl_array_t *queue_external_cis(jl_array_t *list) } // New roots for external methods -static void jl_collect_methods(htable_t *mset, jl_array_t *new_specializations) +static void jl_collect_new_roots(jl_array_t *roots, jl_array_t *new_specializations, uint64_t key) { - size_t i, l = new_specializations ? jl_array_len(new_specializations) : 0; - jl_value_t *v; - jl_method_t *m; - for (i = 0; i < l; i++) { - v = jl_array_ptr_ref(new_specializations, i); - assert(jl_is_code_instance(v)); - m = ((jl_code_instance_t*)v)->def->def.method; + htable_t mset; + htable_new(&mset, 0); + size_t l = new_specializations ? jl_array_len(new_specializations) : 0; + for (size_t i = 0; i < l; i++) { + jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(new_specializations, i); + assert(jl_is_code_instance(ci)); + jl_method_t *m = ci->def->def.method; assert(jl_is_method(m)); - ptrhash_put(mset, (void*)m, (void*)m); + ptrhash_put(&mset, (void*)m, (void*)m); } -} - -static void jl_collect_new_roots(jl_array_t *roots, const htable_t *mset, uint64_t key) -{ - size_t i, sz = mset->size; int nwithkey; - jl_method_t *m; - void *const *table = mset->table; + void *const *table = mset.table; jl_array_t *newroots = NULL; JL_GC_PUSH1(&newroots); - for (i = 0; i < sz; i += 2) { + for (size_t i = 0; i < mset.size; i += 2) { if (table[i+1] != HT_NOTFOUND) { - m = (jl_method_t*)table[i]; + jl_method_t *m = (jl_method_t*)table[i]; assert(jl_is_method(m)); nwithkey = nroots_with_key(m, key); if (nwithkey) { @@ -305,6 +289,7 @@ static void jl_collect_new_roots(jl_array_t *roots, const htable_t *mset, uint64 } } JL_GC_POP(); + htable_free(&mset); } // Create the forward-edge map (caller => callees) @@ -422,9 +407,18 @@ static void jl_record_edges(jl_method_instance_t *caller, arraylist_t *wq, jl_ar // Extract `edges` and `ext_targets` from `edges_map` // `edges` = [caller1, targets_indexes1, ...], the list of methods and their edges // `ext_targets` is [invokesig1, callee1, matches1, ...], the edges for each target -static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets) +static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *external_cis) { size_t world = jl_atomic_load_acquire(&jl_world_counter); + htable_t external_mis; + htable_new(&external_mis, 0); + if (external_cis) { + for (size_t i = 0; i < jl_array_len(external_cis); i++) { + jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(external_cis, i); + jl_method_instance_t *mi = ci->def; + ptrhash_put(&external_mis, (void*)mi, (void*)mi); + } + } arraylist_t wq; arraylist_new(&wq, 0); void **table = (void**)jl_array_data(edges_map); // edges_map is caller => callees @@ -438,10 +432,11 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets) continue; assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method)); if (!jl_object_in_image((jl_value_t*)caller->def.method->module) || - method_instance_in_queue(caller)) { + ptrhash_get(&external_mis, caller) != HT_NOTFOUND) { jl_record_edges(caller, &wq, edges); } } + htable_free(&external_mis); while (wq.len) { jl_method_instance_t *caller = (jl_method_instance_t*)arraylist_pop(&wq); jl_record_edges(caller, &wq, edges); @@ -1061,6 +1056,7 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_a htable_reset(&visited, l); for (i = 0; i < l; i++) { jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(ci_list, i); + assert(ci->max_world == 1 || ci->max_world == ~(size_t)0); assert(ptrhash_get(&visited, (void*)ci->def) == HT_NOTFOUND); // check that we don't have multiple cis for same mi ptrhash_put(&visited, (void*)ci->def, (void*)ci); } @@ -1121,7 +1117,7 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_a assert(jl_is_code_instance(ci)); jl_code_instance_t *codeinst = (jl_code_instance_t*)ci; remove_code_instance_from_validation(codeinst); // mark it as handled - assert(codeinst->min_world >= world && codeinst->inferred); + assert(codeinst->min_world == world && codeinst->inferred); codeinst->max_world = ~(size_t)0; if (jl_rettype_inferred(caller, world, ~(size_t)0) == jl_nothing) { jl_mi_cache_insert(caller, codeinst); @@ -1161,8 +1157,8 @@ static void validate_new_code_instances(void) //assert(0 && "unexpected unprocessed CodeInstance found"); jl_code_instance_t *ci = (jl_code_instance_t*)new_code_instance_validate.table[i]; JL_GC_PROMISE_ROOTED(ci); // TODO: this needs a root (or restructuring to avoid it) - assert(ci->min_world >= world && ci->inferred); - ci->max_world = ~(size_t)0; + assert(ci->min_world == world && ci->inferred); + assert(ci->max_world == ~(size_t)0); jl_method_instance_t *caller = ci->def; if (jl_rettype_inferred(caller, world, ~(size_t)0) == jl_nothing) { jl_mi_cache_insert(caller, ci); diff --git a/test/precompile.jl b/test/precompile.jl index 0febfecb78b69..dbe40f3ba6204 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1686,64 +1686,73 @@ end precompile_test_harness("DynamicExpressions") do load_path # https://github.com/JuliaLang/julia/pull/47184#issuecomment-1364716312 write(joinpath(load_path, "Float16MWE.jl"), - """ - module Float16MWE - struct Node{T} - val::T - end - doconvert(::Type{<:Node}, val) = convert(Float16, val) - precompile(Tuple{typeof(doconvert), Type{Node{Float16}}, Float64}) - end # module Float16MWE - """) + """ + module Float16MWE + struct Node{T} + val::T + end + doconvert(::Type{<:Node}, val) = convert(Float16, val) + precompile(Tuple{typeof(doconvert), Type{Node{Float16}}, Float64}) + end # module Float16MWE + """) Base.compilecache(Base.PkgId("Float16MWE")) - (@eval (using Float16MWE)) - Base.invokelatest() do - @test Float16MWE.doconvert(Float16MWE.Node{Float16}, -1.2) === Float16(-1.2) - end + @eval using Float16MWE + @test @invokelatest(Float16MWE.doconvert(Float16MWE.Node{Float16}, -1.2)) === Float16(-1.2) end precompile_test_harness("BadInvalidations") do load_path write(joinpath(load_path, "BadInvalidations.jl"), - """ - module BadInvalidations + """ + module BadInvalidations Base.Experimental.@compiler_options compile=min optimize=1 getval() = Base.a_method_to_overwrite_in_test() getval() - end # module BadInvalidations - """) + end # module BadInvalidations + """) Base.compilecache(Base.PkgId("BadInvalidations")) - (@eval Base a_method_to_overwrite_in_test() = inferencebarrier(2)) - (@eval (using BadInvalidations)) - Base.invokelatest() do - @test BadInvalidations.getval() === 2 - end + @eval Base a_method_to_overwrite_in_test() = inferencebarrier(2) + @eval using BadInvalidations + @test Base.invokelatest(BadInvalidations.getval) === 2 end # https://github.com/JuliaLang/julia/issues/48074 precompile_test_harness("WindowsCacheOverwrite") do load_path # https://github.com/JuliaLang/julia/pull/47184#issuecomment-1364716312 write(joinpath(load_path, "WindowsCacheOverwrite.jl"), - """ - module WindowsCacheOverwrite - - end # module - """) + """ + module WindowsCacheOverwrite + end # module + """) ji, ofile = Base.compilecache(Base.PkgId("WindowsCacheOverwrite")) - (@eval (using WindowsCacheOverwrite)) + @eval using WindowsCacheOverwrite write(joinpath(load_path, "WindowsCacheOverwrite.jl"), - """ - module WindowsCacheOverwrite - - f() = "something new" - - end # module - """) + """ + module WindowsCacheOverwrite + f() = "something new" + end # module + """) ji_2, ofile_2 = Base.compilecache(Base.PkgId("WindowsCacheOverwrite")) @test ofile_2 == Base.ocachefile_from_cachefile(ji_2) end +precompile_test_harness("Issue #48391") do load_path + write(joinpath(load_path, "I48391.jl"), + """ + module I48391 + struct SurrealFinite <: Real end + precompile(Tuple{typeof(Base.isless), SurrealFinite, SurrealFinite}) + Base.:(<)(x::SurrealFinite, y::SurrealFinite) = "good" + end + """) + ji, ofile = Base.compilecache(Base.PkgId("I48391")) + @eval using I48391 + x = Base.invokelatest(I48391.SurrealFinite) + @test Base.invokelatest(isless, x, x) === "good" + @test_throws ErrorException isless(x, x) +end + empty!(Base.DEPOT_PATH) append!(Base.DEPOT_PATH, original_depot_path) empty!(Base.LOAD_PATH)