Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

account for extensions indirectly depending on each other in parallel package precompilation #53972

Merged
merged 1 commit into from
May 9, 2024
Merged

account for extensions indirectly depending on each other in parallel package precompilation #53972

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 49 additions & 7 deletions base/precompilation.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,6 @@ function precompilepkgs(pkgs::Vector{String}=String[];
depsmap[pkg] = filter!(!Base.in_sysimage, deps)
# add any extensions
pkg_exts = Dict{Base.PkgId, Vector{Base.PkgId}}()
prev_ext = nothing
for (ext_name, extdep_uuids) in env.extensions[dep]
ext_deps = Base.PkgId[]
push!(ext_deps, pkg) # depends on parent package
Expand All @@ -417,13 +416,8 @@ function precompilepkgs(pkgs::Vector{String}=String[];
end
end
all_extdeps_available || continue
if prev_ext isa Base.PkgId
# also make the exts depend on eachother sequentially to avoid race
push!(ext_deps, prev_ext)
end
ext_uuid = Base.uuid5(pkg.uuid, ext_name)
ext = Base.PkgId(ext_uuid, ext_name)
prev_ext = ext
filter!(!Base.in_sysimage, ext_deps)
depsmap[ext] = ext_deps
exts[ext] = pkg.name
Expand All @@ -434,6 +428,51 @@ function precompilepkgs(pkgs::Vector{String}=String[];
end
end

# An extension effectively depends on another extension if it has all the the
# dependencies of that other extension
function expand_dependencies(depsmap)
function visit!(visited, node, all_deps)
if node in visited
return
end
push!(visited, node)
for dep in get(Set{Base.PkgId}, depsmap, node)
if !(dep in all_deps)
push!(all_deps, dep)
visit!(visited, dep, all_deps)
end
end
end

depsmap_transitive = Dict{Base.PkgId, Set{Base.PkgId}}()
for package in keys(depsmap)
# Initialize a set to keep track of all dependencies for 'package'
all_deps = Set{Base.PkgId}()
visited = Set{Base.PkgId}()
visit!(visited, package, all_deps)
# Update depsmap with the complete set of dependencies for 'package'
depsmap_transitive[package] = all_deps
end
return depsmap_transitive
end

depsmap_transitive = expand_dependencies(depsmap)

for (_, extensions_1) in pkg_exts_map
for extension_1 in extensions_1
deps_ext_1 = depsmap_transitive[extension_1]
for (_, extensions_2) in pkg_exts_map
for extension_2 in extensions_2
extension_1 == extension_2 && continue
deps_ext_2 = depsmap_transitive[extension_2]
if issubset(deps_ext_2, deps_ext_1)
push!(depsmap[extension_1], extension_2)
end
end
end
end
end

@debug "precompile: deps collected"
# this loop must be run after the full depsmap has been populated
for (pkg, pkg_exts) in pkg_exts_map
Expand Down Expand Up @@ -748,6 +787,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
end
@debug "precompile: starting precompilation loop" depsmap direct_deps
## precompilation loop

for (pkg, deps) in depsmap
cachepaths = Base.find_all_in_cache_path(pkg)
sourcepath = Base.locate_package(pkg)
Expand Down Expand Up @@ -820,6 +860,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
end
loaded && (n_loaded += 1)
catch err
# @show err
close(std_pipe.in) # close pipe to end the std output monitor
wait(t_monitor)
if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file"))
Expand All @@ -843,7 +884,8 @@ function precompilepkgs(pkgs::Vector{String}=String[];
notify(was_processed[pkg_config])
catch err_outer
# For debugging:
# println("Task failed $err_outer") # logging doesn't show here
# println("Task failed $err_outer")
# Base.display_error(ErrorException(""), Base.catch_backtrace())# logging doesn't show here
handle_interrupt(err_outer) || rethrow()
notify(was_processed[pkg_config])
finally
Expand Down