Skip to content

Commit

Permalink
Use Base parallel precompilation to build stdlibs (JuliaLang#53598)
Browse files Browse the repository at this point in the history
Follow-on from JuliaLang#53403

This extends `Base.Precompilation.precompilepkgs` to take a list of
configurations to precompile each package with, while parallelizing
across all packages and configurations, and uses it to build the stdlib
pkgimages.

It simplifies the stdlib pkgimage build process but is (currently)
dependent on having an accurately resolved Manifest.toml (Project.toml
included to make the manifest easier to make). Any new/removed stdlibs
or changes their dependencies will require updating the Manifest.toml.
It's a bit chicken and egg, but should be manageable with manual editing
of the Manifest.toml.

In terms of speed improvement:
MacOS aarch64 CI runner 6m19s before, 5m19 with this

Note that CI builds will show the basic print with timing of each
package, whereas local build will be the tidier fancy print without
timings.

Co-authored-by: Valentin Churavy <vchuravy@users.noreply.github.com>
  • Loading branch information
2 people authored and mkitti committed Apr 13, 2024
1 parent e7166a6 commit 3938502
Show file tree
Hide file tree
Showing 9 changed files with 589 additions and 291 deletions.
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink j
julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest julia-base-cache

stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-%
@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f pkgimage.mk all-$*
@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f pkgimage.mk $*

debug release : % : julia-% stdlibs-cache-%

Expand Down Expand Up @@ -588,6 +588,7 @@ clean: | $(CLEAN_TARGETS)
@-$(MAKE) -C $(BUILDROOT)/cli clean
@-$(MAKE) -C $(BUILDROOT)/test clean
@-$(MAKE) -C $(BUILDROOT)/stdlib clean
@-$(MAKE) -C $(BUILDROOT) -f pkgimage.mk clean
-rm -f $(BUILDROOT)/julia
-rm -f $(BUILDROOT)/*.tar.gz
-rm -f $(build_depsbindir)/stringreplace \
Expand Down
33 changes: 26 additions & 7 deletions base/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1524,7 +1524,21 @@ function CacheFlags(f::UInt8)
CacheFlags(use_pkgimages, debug_level, check_bounds, inline, opt_level)
end
CacheFlags(f::Int) = CacheFlags(UInt8(f))
CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ()))
function CacheFlags(cf::CacheFlags=CacheFlags(ccall(:jl_cache_flags, UInt8, ()));
use_pkgimages::Union{Nothing,Bool}=nothing,
debug_level::Union{Nothing,Int}=nothing,
check_bounds::Union{Nothing,Int}=nothing,
inline::Union{Nothing,Bool}=nothing,
opt_level::Union{Nothing,Int}=nothing
)
return CacheFlags(
use_pkgimages === nothing ? cf.use_pkgimages : use_pkgimages,
debug_level === nothing ? cf.debug_level : debug_level,
check_bounds === nothing ? cf.check_bounds : check_bounds,
inline === nothing ? cf.inline : inline,
opt_level === nothing ? cf.opt_level : opt_level
)
end

function _cacheflag_to_uint8(cf::CacheFlags)::UInt8
f = UInt8(0)
Expand Down Expand Up @@ -2768,7 +2782,7 @@ function compilecache_dir(pkg::PkgId)
return joinpath(DEPOT_PATH[1], entrypath)
end

function compilecache_path(pkg::PkgId, prefs_hash::UInt64; project::String=something(Base.active_project(), ""))::String
function compilecache_path(pkg::PkgId, prefs_hash::UInt64; flags::CacheFlags=CacheFlags(), project::String=something(Base.active_project(), ""))::String
entrypath, entryfile = cache_file_entry(pkg)
cachepath = joinpath(DEPOT_PATH[1], entrypath)
isdir(cachepath) || mkpath(cachepath)
Expand All @@ -2778,7 +2792,7 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64; project::String=somet
crc = _crc32c(project)
crc = _crc32c(unsafe_string(JLOptions().image_file), crc)
crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc)
crc = _crc32c(ccall(:jl_cache_flags, UInt8, ()), crc)
crc = _crc32c(_cacheflag_to_uint8(flags), crc)

cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing)
if cpu_target === nothing
Expand Down Expand Up @@ -2810,7 +2824,8 @@ end
const MAX_NUM_PRECOMPILE_FILES = Ref(10)

function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, internal_stdout::IO = stdout,
keep_loaded_modules::Bool = true; flags::Cmd=``, reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}())
keep_loaded_modules::Bool = true; flags::Cmd=``, cacheflags::CacheFlags=CacheFlags(),
reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}())

@nospecialize internal_stderr internal_stdout
# decide where to put the resulting cache file
Expand Down Expand Up @@ -2859,7 +2874,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in
# Read preferences hash back from .ji file (we can't precompute because
# we don't actually know what the list of compile-time preferences are without compiling)
prefs_hash = preferences_hash(tmppath)
cachefile = compilecache_path(pkg, prefs_hash)
cachefile = compilecache_path(pkg, prefs_hash; flags=cacheflags)
ocachefile = cache_objects ? ocachefile_from_cachefile(cachefile) : nothing

# append checksum for so to the end of the .ji file:
Expand Down Expand Up @@ -3408,7 +3423,7 @@ global parse_pidfile_hook
# The preferences hash is only known after precompilation so just assume no preferences.
# Also ignore the active project, which means that if all other conditions are equal,
# the same package cannot be precompiled from different projects and/or different preferences at the same time.
compilecache_pidfile_path(pkg::PkgId) = compilecache_path(pkg, UInt64(0); project="") * ".pidfile"
compilecache_pidfile_path(pkg::PkgId; flags::CacheFlags=CacheFlags()) = compilecache_path(pkg, UInt64(0); project="", flags) * ".pidfile"

const compilecache_pidlock_stale_age = 10

Expand Down Expand Up @@ -3536,12 +3551,16 @@ end
M = root_module(req_key)
if PkgId(M) == req_key && module_build_id(M) === req_build_id
depmods[i] = M
elseif M == Core
@debug "Rejecting cache file $cachefile because it was made with a different julia version"
record_reason(reasons, "wrong julia version")
return true # Won't be able to fulfill dependency
elseif ignore_loaded || !stalecheck
# Used by Pkg.precompile given that there it's ok to precompile different versions of loaded packages
@goto locate_branch
else
@debug "Rejecting cache file $cachefile because module $req_key is already loaded and incompatible."
record_reason(reasons, req_key == PkgId(Core) ? "wrong julia version" : "wrong dep version loaded")
record_reason(reasons, "wrong dep version loaded")
return true # Won't be able to fulfill dependency
end
else
Expand Down
Loading

0 comments on commit 3938502

Please sign in to comment.