diff --git a/src/codegen.cpp b/src/codegen.cpp index 18a0467104706f..07b935b1449a15 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7549,7 +7549,7 @@ static std::pair, jl_llvm_functions_t> break; } if (j == jlen) // not found - add to array - jl_array_ptr_1d_push(m->roots, ival); + jl_add_method_root(m, jl_precompile_toplevel_module, ival); } } ctx.roots = NULL; diff --git a/src/dump.c b/src/dump.c index 0d07e46c8a0c7e..afc372696a8f2b 100644 --- a/src/dump.c +++ b/src/dump.c @@ -704,6 +704,7 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li write_int8(s->s, m->constprop); jl_serialize_value(s, (jl_value_t*)m->slot_syms); jl_serialize_value(s, (jl_value_t*)m->roots); + jl_serialize_value(s, (jl_value_t*)m->root_blocks); jl_serialize_value(s, (jl_value_t*)m->ccallable); jl_serialize_value(s, (jl_value_t*)m->source); jl_serialize_value(s, (jl_value_t*)m->unspecialized); @@ -1573,6 +1574,9 @@ static jl_value_t *jl_deserialize_value_method(jl_serializer_state *s, jl_value_ m->roots = (jl_array_t*)jl_deserialize_value(s, (jl_value_t**)&m->roots); if (m->roots) jl_gc_wb(m, m->roots); + m->root_blocks = (jl_array_t*)jl_deserialize_value(s, (jl_value_t**)&m->root_blocks); + if (m->root_blocks) + jl_gc_wb(m, m->root_blocks); m->ccallable = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&m->ccallable); if (m->ccallable) { jl_gc_wb(m, m->ccallable); @@ -2312,6 +2316,8 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) } JL_GC_PUSH2(&mod_array, &udeps); mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) + assert(jl_precompile_toplevel_module == NULL); + jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); serializer_worklist = worklist; write_header(&f); @@ -2426,6 +2432,7 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) write_int32(&f, 0); // mark the end of the source text ios_close(&f); JL_GC_POP(); + jl_precompile_toplevel_module = NULL; return 0; } diff --git a/src/init.c b/src/init.c index 1eb2cccd73d682..521a2fa80613a0 100644 --- a/src/init.c +++ b/src/init.c @@ -625,6 +625,7 @@ JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) libsupport_init(); htable_new(&jl_current_modules, 0); JL_MUTEX_INIT(&jl_modules_mutex); + jl_precompile_toplevel_module = NULL; ios_set_io_wait_func = jl_set_io_wait; jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.), // best to call this first, since it also initializes libuv diff --git a/src/ircode.c b/src/ircode.c index 279e458728e243..3bb60eb7b24d27 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -48,7 +48,7 @@ static int literal_val_id(jl_ircode_state *s, jl_value_t *v) JL_GC_DISABLED return i; } } - jl_array_ptr_1d_push(rs, v); + jl_add_method_root(s->method, jl_precompile_toplevel_module, v); return jl_array_len(rs) - 1; } diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 588d2a831e2254..2bfd53348d190d 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -18,6 +18,7 @@ XX(jl_array_type) \ XX(jl_array_typename) \ XX(jl_array_uint8_type) \ + XX(jl_array_uint64_type) \ XX(jl_atomicerror_type) \ XX(jl_base_module) \ XX(jl_bool_type) \ @@ -84,6 +85,7 @@ XX(jl_pinode_type) \ XX(jl_pointer_type) \ XX(jl_pointer_typename) \ + XX(jl_precompile_toplevel_module) \ XX(jl_quotenode_type) \ XX(jl_readonlymemory_exception) \ XX(jl_ref_type) \ diff --git a/src/jltypes.c b/src/jltypes.c index cfed8982cbc7c5..00423886603627 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2253,6 +2253,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_array_symbol_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_symbol_type, jl_box_long(1)); jl_array_uint8_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_uint8_type, jl_box_long(1)); jl_array_int32_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_int32_type, jl_box_long(1)); + jl_array_uint64_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_uint64_type, jl_box_long(1)); jl_an_empty_vec_any = (jl_value_t*)jl_alloc_vec_any(0); // used internally jl_atomic_store_relaxed(&jl_nonfunction_mt->leafcache, (jl_array_t*)jl_an_empty_vec_any); jl_atomic_store_relaxed(&jl_type_type_mt->leafcache, (jl_array_t*)jl_an_empty_vec_any); @@ -2392,7 +2393,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_method_type = jl_new_datatype(jl_symbol("Method"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(26, + jl_perm_symsvec(27, "name", "module", "file", @@ -2408,6 +2409,7 @@ void jl_init_types(void) JL_GC_DISABLED "unspecialized", // !const "generator", // !const "roots", // !const + "root_blocks", // !const "ccallable", // !const "invokes", // !const "recursion_relation", // !const @@ -2419,7 +2421,7 @@ void jl_init_types(void) JL_GC_DISABLED "pure", "is_for_opaque_closure", "constprop"), - jl_svec(26, + jl_svec(27, jl_symbol_type, jl_module_type, jl_symbol_type, @@ -2435,6 +2437,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type, // jl_method_instance_type jl_any_type, jl_array_any_type, + jl_array_uint64_type, jl_simplevector_type, jl_any_type, jl_any_type, diff --git a/src/julia.h b/src/julia.h index 34ecd153893991..27f9842e5f3192 100644 --- a/src/julia.h +++ b/src/julia.h @@ -291,6 +291,9 @@ typedef struct _jl_method_t { _Atomic(struct _jl_method_instance_t*) unspecialized; // unspecialized executable method instance, or null jl_value_t *generator; // executable code-generating function if available jl_array_t *roots; // pointers in generated code (shared to reduce memory), or null + // Identify roots by module-of-origin. We only track the module for roots added during incremental compilation. + // May be NULL if no external roots have been added, otherwise it's a Vector{UInt64} + jl_array_t *root_blocks; // RLE (build_id, offset) pairs (even/odd indexing) jl_svec_t *ccallable; // svec(rettype, sig) if a ccallable entry point is requested for this // cache of specializations of this method for invoke(), i.e. @@ -708,6 +711,7 @@ extern JL_DLLIMPORT jl_value_t *jl_array_uint8_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_any_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_symbol_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_int32_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_array_uint64_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_expr_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_globalref_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_linenumbernode_type JL_GLOBALLY_ROOTED; diff --git a/src/julia_internal.h b/src/julia_internal.h index 1fbfb082e4a8f1..547d0a1cb12a4f 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -527,6 +527,8 @@ JL_DLLEXPORT jl_code_info_t *jl_new_code_info_uninit(void); void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals, int binding_effects); +JL_DLLEXPORT void jl_add_method_root(jl_method_t *m, jl_module_t *mod, jl_value_t* root); + int jl_valid_type_param(jl_value_t *v); JL_DLLEXPORT jl_value_t *jl_apply_2va(jl_value_t *f, jl_value_t **args, uint32_t nargs); @@ -647,6 +649,7 @@ jl_binding_t *jl_get_module_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t JL_DLLEXPORT void jl_binding_deprecation_warning(jl_module_t *m, jl_binding_t *b); extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED; extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; +extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; int jl_compile_extern_c(void *llvmmod, void *params, void *sysimg, jl_value_t *declrt, jl_value_t *sigt); jl_opaque_closure_t *jl_new_opaque_closure(jl_tupletype_t *argt, jl_value_t *isva, jl_value_t *rt_lb, diff --git a/src/method.c b/src/method.c index d8cd1c30a94e18..2d39b7e5923d94 100644 --- a/src/method.c +++ b/src/method.c @@ -731,6 +731,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) m->sig = NULL; m->slot_syms = NULL; m->roots = NULL; + m->root_blocks = NULL; m->ccallable = NULL; m->module = module; m->external_mt = NULL; @@ -987,6 +988,53 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, return m; } +// root blocks + +static uint64_t current_root_id(jl_array_t *root_blocks) +{ + if (!root_blocks) + return 0; + assert(jl_is_array(root_blocks)); + size_t nx2 = jl_array_len(root_blocks); + if (nx2 == 0) + return 0; + uint64_t *blocks = (uint64_t*)jl_array_data(root_blocks); + return blocks[nx2-2]; +} + +static void add_root_block(jl_array_t *root_blocks, uint64_t modid, size_t len) +{ + assert(jl_is_array(root_blocks)); + jl_array_grow_end(root_blocks, 2); + uint64_t *blocks = (uint64_t*)jl_array_data(root_blocks); + int nx2 = jl_array_len(root_blocks); + blocks[nx2-2] = modid; + blocks[nx2-1] = len; +} + +JL_DLLEXPORT void jl_add_method_root(jl_method_t *m, jl_module_t *mod, jl_value_t* root) +{ + JL_GC_PUSH2(&m, &root); + uint64_t modid = 0; + if (mod) { + assert(jl_is_module(mod)); + modid = mod->build_id; + } + assert(jl_is_method(m)); + if (!m->roots) { + m->roots = jl_alloc_vec_any(0); + jl_gc_wb(m, m->roots); + } + if (!m->root_blocks && modid != 0) { + m->root_blocks = jl_alloc_array_1d(jl_array_uint64_type, 0); + jl_gc_wb(m, m->root_blocks); + } + if (current_root_id(m->root_blocks) != modid) + add_root_block(m->root_blocks, modid, jl_array_len(m->roots)); + jl_array_ptr_1d_push(m->roots, root); + JL_GC_POP(); +} + #ifdef __cplusplus } #endif diff --git a/src/staticdata.c b/src/staticdata.c index 634728991a3586..37415a234808ce 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -26,7 +26,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 151 +#define NUM_TAGS 152 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -106,6 +106,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_array_symbol_type); INSERT_TAG(jl_array_uint8_type); INSERT_TAG(jl_array_int32_type); + INSERT_TAG(jl_array_uint64_type); INSERT_TAG(jl_int32_type); INSERT_TAG(jl_int64_type); INSERT_TAG(jl_bool_type); diff --git a/src/toplevel.c b/src/toplevel.c index 363d44b3cd6423..1d2bac883206c1 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -33,6 +33,9 @@ JL_DLLEXPORT const char *jl_filename = "none"; // need to update jl_critical_err htable_t jl_current_modules; jl_mutex_t jl_modules_mutex; +// During incremental compilation, the following gets set +JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module; // the toplevel module currently being defined + JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m) { jl_module_t *base_module = jl_base_relative_to(m); @@ -138,11 +141,16 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex ptrhash_put(&jl_current_modules, (void*)newm, (void*)((uintptr_t)HT_NOTFOUND + 1)); JL_UNLOCK(&jl_modules_mutex); + jl_module_t *old_toplevel_module = jl_precompile_toplevel_module; + // copy parent environment info into submodule newm->uuid = parent_module->uuid; if (jl_is__toplevel__mod(parent_module)) { newm->parent = newm; jl_register_root_module(newm); + if (jl_options.incremental) { + jl_precompile_toplevel_module = newm; + } } else { newm->parent = parent_module; @@ -261,6 +269,8 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex } } + jl_precompile_toplevel_module = old_toplevel_module; + JL_GC_POP(); return (jl_value_t*)newm; } diff --git a/test/precompile.jl b/test/precompile.jl index 999bd07c9e12b6..06ff7516b0b4a2 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -33,6 +33,56 @@ function precompile_test_harness(@nospecialize(f), separate::Bool) nothing end +# method root provenance + +rootid(m::Module) = ccall(:jl_module_build_id, UInt64, (Any,), Base.parentmodule(m)) +rootid(m::Method) = rootid(m.module) + +function root_provenance(m::Method, i::Int) + mid = rootid(m) + isdefined(m, :root_blocks) || return mid + idxs = view(m.root_blocks, 2:2:length(m.root_blocks)) + j = searchsortedfirst(idxs, i) - 1 # RLE roots are 0-indexed + j == 0 && return mid + return m.root_blocks[2*j-1] +end + +struct RLEIterator{T} # for method roots, T = UInt64 (even on 32-bit) + items::Vector{Any} + blocks::Vector{T} + defaultid::T +end +function RLEIterator(roots, blocks, defaultid) + T = promote_type(eltype(blocks), typeof(defaultid)) + return RLEIterator{T}(convert(Vector{Any}, roots), blocks, defaultid) +end +RLEIterator(m::Method) = RLEIterator(m.roots, m.root_blocks, rootid(m)) +Base.iterate(iter::RLEIterator) = iterate(iter, (0, 0, iter.defaultid)) +function Base.iterate(iter::RLEIterator, (i, j, cid)) + i += 1 + i > length(iter.items) && return nothing + r = iter.items[i] + while (j + 1 < length(iter.blocks) && i > iter.blocks[j+2]) + cid = iter.blocks[j+1] + j += 2 + end + return cid => r, (i, j, cid) +end + +function group_roots(m::Method) + mid = rootid(m) + isdefined(m, :root_blocks) || return Dict(mid => m.roots) + group_roots(RLEIterator(m.roots, m.root_blocks, mid)) +end +function group_roots(iter::RLEIterator) + rootsby = Dict{typeof(iter.defaultid),Vector{Any}}() + for (id, r) in iter + list = get!(valtype(rootsby), rootsby, id) + push!(list, r) + end + return rootsby +end + precompile_test_harness("basic precompile functionality") do dir2 precompile_test_harness(false) do dir @@ -535,6 +585,87 @@ precompile_test_harness(false) do dir end end +# method root provenance +# setindex!(::Dict{K,V}, ::Any, ::K) adds both compression and codegen roots +precompile_test_harness("code caching") do dir + Bid = rootid(Base) + Cache_module = :Cacheb8321416e8a3e2f1 + write(joinpath(dir, "$Cache_module.jl"), + """ + module $Cache_module + struct X end + struct X2 end + @noinline function f(d) + @noinline + d[X()] = nothing + end + @noinline fpush(dest) = push!(dest, X()) + function callboth() + f(Dict{X,Any}()) + fpush(X[]) + nothing + end + precompile(callboth, ()) + end + """) + Base.compilecache(Base.PkgId(string(Cache_module))) + @eval using $Cache_module + M = getfield(@__MODULE__, Cache_module) + Mid = rootid(M) + for name in (:f, :fpush, :callboth) + func = getfield(M, name) + m = only(collect(methods(func))) + @test all(i -> root_provenance(m, i) == Mid, 1:length(m.roots)) + end + m = which(setindex!, (Dict{M.X,Any}, Any, M.X)) + @test_broken M.X ∈ m.roots # requires caching external compilation results + Base.invokelatest() do + Dict{M.X2,Any}()[M.X2()] = nothing + end + @test M.X2 ∈ m.roots + groups = group_roots(m) + @test_broken M.X ∈ groups[Mid] # requires caching external compilation results + @test M.X2 ∈ groups[rootid(@__MODULE__)] + @test !isempty(groups[Bid]) + # PkgA loads PkgB, and both add roots to the same method (both before and after loading B) + Cache_module2 = :Cachea1544c83560f0c99 + write(joinpath(dir, "$Cache_module2.jl"), + """ + module $Cache_module2 + struct Y end + @noinline f(dest) = push!(dest, Y()) + callf() = f(Y[]) + callf() + using $(Cache_module) + struct Z end + @noinline g(dest) = push!(dest, Z()) + callg() = g(Z[]) + callg() + end + """) + Base.compilecache(Base.PkgId(string(Cache_module2))) + @eval using $Cache_module2 + M2 = getfield(@__MODULE__, Cache_module2) + M2id = rootid(M2) + dest = [] + Base.invokelatest() do # use invokelatest to see the results of loading the compile + M2.f(dest) + M.fpush(dest) + M2.g(dest) + @test dest == [M2.Y(), M.X(), M2.Z()] + @test M2.callf() == [M2.Y()] + @test M2.callg() == [M2.Z()] + @test M.fpush(M.X[]) == [M.X()] + end + mT = which(push!, (Vector{T} where T, Any)) + groups = group_roots(mT) + # all below require caching external CodeInstances + @test_broken M2.Y ∈ groups[M2id] + @test_broken M2.Z ∈ groups[M2id] + @test_broken M.X ∈ groups[Mid] + @test_broken M.X ∉ groups[M2id] +end + # test --compiled-modules=no command line option precompile_test_harness("--compiled-modules=no") do dir Time_module = :Time4b3a94a1a081a8cb