Skip to content

Commit

Permalink
add logic to prefer loading modules that are already loaded
Browse files Browse the repository at this point in the history 
Iterate over the list of existing loaded modules for PkgId whenever
loading a new module for PkgId, so that we will use that existing
build_id content if it otherwise passes the other stale_checks.
  • Loading branch information
@vtjnash
vtjnash committed Sep 30, 2024
1 parent 4da0671 commit 83ed90f
Show file tree
Hide file tree
Showing 3 changed files with 134 additions and 82 deletions.
2 changes: 1 addition & 1 deletion base/Base.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -649,7 +649,7 @@ function __init__()
empty!(explicit_loaded_modules)
empty!(loaded_precompiles) # If we load a packageimage when building the image this might not be empty
for (mod, key) in module_keys
loaded_precompiles[key => module_build_id(mod)] = mod
push!(get!(Vector{Module}, loaded_precompiles, key), mod)
end
if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES")
MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"])
Expand Down
184 changes: 103 additions & 81 deletions base/loading.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1231,7 +1231,7 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No
dep = depmods[i]
dep isa Module && continue
_, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128}
dep = loaded_precompiles[depkey => depbuild_id]
dep = something(maybe_loaded_precompile(depkey, depbuild_id))
@assert PkgId(dep) == depkey && module_build_id(dep) === depbuild_id
depmods[i] = dep
end
Expand Down Expand Up @@ -1337,6 +1337,7 @@ end

function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
# This function is also used by PkgCacheInspector.jl
assert_havelock(require_lock)
restored = sv[1]::Vector{Any}
for M in restored
M = M::Module
Expand All @@ -1345,7 +1346,7 @@ function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
end
if parentmodule(M) === M
push!(loaded_modules_order, M)
loaded_precompiles[pkg => module_build_id(M)] = M
push!(get!(Vector{Module}, loaded_precompiles, pkg), M)
end
end

Expand Down Expand Up @@ -1945,90 +1946,102 @@ end
assert_havelock(require_lock)
paths = find_all_in_cache_path(pkg, DEPOT_PATH)
newdeps = PkgId[]
for path_to_try in paths::Vector{String}
staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons, stalecheck)
if staledeps === true
continue
end
try
staledeps, ocachefile, newbuild_id = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
# finish checking staledeps module graph
for i in eachindex(staledeps)
dep = staledeps[i]
dep isa Module && continue
modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128}
modpaths = find_all_in_cache_path(modkey, DEPOT_PATH)
for modpath_to_try in modpaths
modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try; stalecheck)
if modstaledeps === true
continue
end
modstaledeps, modocachepath, _ = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath)
@goto check_next_dep
try_build_ids = UInt128[build_id]
if build_id == UInt128(0)
let loaded = get(loaded_precompiles, pkg, nothing)
if loaded !== nothing
for mod in loaded # try these in reverse original load order to see if one is already valid
pushfirst!(try_build_ids, module_build_id(mod))
end
@debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache."
@goto check_next_path
@label check_next_dep
end
M = get(loaded_precompiles, pkg => newbuild_id, nothing)
if isa(M, Module)
stalecheck && register_root_module(M)
return M
end
if stalecheck
try
touch(path_to_try) # update timestamp of precompilation file
catch ex # file might be read-only and then we fail to update timestamp, which is fine
ex isa IOError || rethrow()
end
end
end
for build_id in try_build_ids
for path_to_try in paths::Vector{String}
staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons, stalecheck)
if staledeps === true
continue
end
# finish loading module graph into staledeps
# TODO: call all start_loading calls (in reverse order) before calling any _include_from_serialized, since start_loading will drop the loading lock
for i in eachindex(staledeps)
dep = staledeps[i]
dep isa Module && continue
modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}}
dep = start_loading(modkey, modbuild_id, stalecheck)
while true
if dep isa Module
if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id
break
else
@debug "Rejecting cache file $path_to_try because module $modkey got loaded at a different version than expected."
@goto check_next_path
try
staledeps, ocachefile, newbuild_id = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
# finish checking staledeps module graph
for i in eachindex(staledeps)
dep = staledeps[i]
dep isa Module && continue
modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128}
modpaths = find_all_in_cache_path(modkey, DEPOT_PATH)
for modpath_to_try in modpaths
modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try; stalecheck)
if modstaledeps === true
continue
end
modstaledeps, modocachepath, _ = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath)
@goto check_next_dep
end
@debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache."
@goto check_next_path
@label check_next_dep
end
M = maybe_loaded_precompile(pkg, newbuild_id)
if isa(M, Module)
stalecheck && register_root_module(M)
return M
end
if stalecheck
try
touch(path_to_try) # update timestamp of precompilation file
catch ex # file might be read-only and then we fail to update timestamp, which is fine
ex isa IOError || rethrow()
end
if dep === nothing
try
set_pkgorigin_version_path(modkey, modpath)
dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps; register = stalecheck)
finally
end_loading(modkey, dep)
end
# finish loading module graph into staledeps
# TODO: call all start_loading calls (in reverse order) before calling any _include_from_serialized, since start_loading will drop the loading lock
for i in eachindex(staledeps)
dep = staledeps[i]
dep isa Module && continue
modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}}
dep = start_loading(modkey, modbuild_id, stalecheck)
while true
if dep isa Module
if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id
break
else
@debug "Rejecting cache file $path_to_try because module $modkey got loaded at a different version than expected."
@goto check_next_path
end
end
if !isa(dep, Module)
@debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep
@goto check_next_path
else
push!(newdeps, modkey)
if dep === nothing
try
set_pkgorigin_version_path(modkey, modpath)
dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps; register = stalecheck)
finally
end_loading(modkey, dep)
end
if !isa(dep, Module)
@debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep
@goto check_next_path
else
push!(newdeps, modkey)
end
end
end
staledeps[i] = dep
end
staledeps[i] = dep
end
restored = get(loaded_precompiles, pkg => newbuild_id, nothing)
if !isa(restored, Module)
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps; register = stalecheck)
end
isa(restored, Module) && return restored
@debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
@label check_next_path
finally
for modkey in newdeps
insert_extension_triggers(modkey)
stalecheck && run_package_callbacks(modkey)
restored = maybe_loaded_precompile(pkg, newbuild_id)
if !isa(restored, Module)
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps; register = stalecheck)
end
isa(restored, Module) && return restored
@debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
@label check_next_path
finally
for modkey in newdeps
insert_extension_triggers(modkey)
stalecheck && run_package_callbacks(modkey)
end
empty!(newdeps)
end
empty!(newdeps)
end
end
return nothing
Expand All @@ -2047,7 +2060,7 @@ function start_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool)
loaded = stalecheck ? maybe_root_module(modkey) : nothing
loaded isa Module && return loaded
if build_id != UInt128(0)
loaded = get(loaded_precompiles, modkey => build_id, nothing)
loaded = maybe_loaded_precompile(modkey, build_id)
loaded isa Module && return loaded
end
loading = get(package_locks, modkey, nothing)
Expand Down Expand Up @@ -2377,12 +2390,21 @@ const pkgorigins = Dict{PkgId,PkgOrigin}()

const explicit_loaded_modules = Dict{PkgId,Module}() # Emptied on Julia start
const loaded_modules = Dict{PkgId,Module}() # available to be explicitly loaded
const loaded_precompiles = Dict{Pair{PkgId,UInt128},Module}() # extended (complete) list of modules, available to be loaded
const loaded_precompiles = Dict{PkgId,Vector{Module}}() # extended (complete) list of modules, available to be loaded
const loaded_modules_order = Vector{Module}()
const module_keys = IdDict{Module,PkgId}() # the reverse of loaded_modules

root_module_key(m::Module) = @lock require_lock module_keys[m]

function maybe_loaded_precompile(key::PkgId, buildid::UInt128)
assert_havelock(require_lock)
mods = get(loaded_precompiles, key, nothing)
mods === nothing && return
for mod in mods
module_build_id(mod) == buildid && return mod
end
end

function module_build_id(m::Module)
hi, lo = ccall(:jl_module_build_id, NTuple{2,UInt64}, (Any,), m)
return (UInt128(hi) << 64) | lo
Expand All @@ -2403,7 +2425,7 @@ end
end
end
end
haskey(loaded_precompiles, key => module_build_id(m)) || push!(loaded_modules_order, m)
maybe_loaded_precompile(key, module_build_id(m)) === nothing && push!(loaded_modules_order, m)
loaded_modules[key] = m
explicit_loaded_modules[key] = m
module_keys[m] = key
Expand Down Expand Up @@ -3783,8 +3805,8 @@ end
for i in 1:ndeps
req_key, req_build_id = required_modules[i]
# Check if module is already loaded
if !stalecheck && haskey(loaded_precompiles, req_key => req_build_id)
M = loaded_precompiles[req_key => req_build_id]
M = stalecheck ? nothing : maybe_loaded_precompile(req_key, req_build_id)
if M !== nothing
@assert PkgId(M) == req_key && module_build_id(M) === req_build_id
depmods[i] = M
elseif root_module_exists(req_key)
Expand Down
30 changes: 30 additions & 0 deletions test/loading.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
original_depot_path = copy(Base.DEPOT_PATH)

using Test
include("precompile_utils.jl")

# Tests for @__LINE__ inside and outside of macros
@test (@__LINE__) == 8
Expand Down Expand Up @@ -1603,3 +1604,32 @@ end
copy!(LOAD_PATH, old_load_path)
end
end

@testset "require_stdlib loading duplication" begin
depot_path = mktempdir()
oldBase64 = nothing
try
push!(empty!(DEPOT_PATH), depot_path)
Base64_key = Base.PkgId(Base.UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"), "Base64")
oldBase64 = Base.unreference_module(Base64_key)
cc = Base.compilecache(Base64_key)
@test Base.isprecompiled(Base64_key, cachepaths=String[cc[1]])
empty!(DEPOT_PATH)
Base.require_stdlib(Base64_key)
push!(DEPOT_PATH, depot_path)
append!(DEPOT_PATH, original_depot_path)
oldloaded = @lock(Base.require_lock, length(get(Base.loaded_precompiles, Base64_key, Module[])))
Base.require(Base64_key)
@test @lock(Base.require_lock, length(get(Base.loaded_precompiles, Base64_key, Module[]))) == oldloaded
Base.unreference_module(Base64_key)
empty!(DEPOT_PATH)
push!(DEPOT_PATH, depot_path)
Base.require(Base64_key)
@test @lock(Base.require_lock, length(get(Base.loaded_precompiles, Base64_key, Module[]))) == oldloaded + 1
Base.unreference_module(Base64_key)
finally
oldBase64 === nothing || Base.register_root_module(oldBase64)
copy!(DEPOT_PATH, original_depot_path)
rm(depot_path, force=true, recursive=true)
end
end

0 comments on commit 83ed90f

Please sign in to comment.