Skip to content

Commit

Permalink
Backports for 1.11.0-rc4/1.11.0 (#55586)
Browse files Browse the repository at this point in the history
Backported PRs:
- [x] #55480 <!-- Fix push! for OffsetVectors, add tests for push! and
append! on AbstractVector -->
- [x] #55443 <!-- Add test for upper/lower/titlecase and fix call -->
- [x] #55524 <!-- Set `.jl` sources as read-only during installation -->
- [x] #55500 <!-- make jl_thread_suspend_and_get_state safe -->
- [x] #55506 <!-- Fix indexing in _mapreducedim for OffsetArrays -->
- [x] #55564 <!-- Empty out loaded_precompiles dict instead of asserting
it's empty. -->
- [x] #55567 <!-- Initialize threadpools correctly during sysimg build
-->
- [x] #55596 <!-- Fast bounds-check for CartesianIndex ranges -->
- [x] #55605 <!-- Reroute Symmetric/Hermitian + Diagonal through
triangular -->
- [x] #55640 <!-- win: move stack_overflow_warning to the backtrace
fiber -->
- [x] #55715 <!-- Add precompile signatures to Markdown to reduce
latency. -->
- [x] #55593 <!-- Fix invalidations for FileIO -->
- [x] #55555 <!-- Revert "Don't expose guard pages to malloc_stack API
consumers" -->
- [x] #55720 <!-- Fix `pkgdir` for extensions -->
- [x] #55729 <!-- Avoid confounding compilation side effects of
`@time_imports` -->
- [x] #55718 <!-- Fix `@time_imports` extension recognition -->
- [x] #55522 <!-- Fix tr for Symmetric/Hermitian block matrices -->

Contains multiple commits, manual intervention needed:
- [ ] #55509 <!-- Fix cong implementation to be properly random and not
just cycling. -->

Non-merged PRs with backport label:
- [ ] #55641 <!-- fall back to slower stat filesize if optimized
filesize fails -->
- [ ] #55534 <!-- Set stdlib sources as read-only during installation
-->
- [ ] #55499 <!-- propagate the terminal's `displaysize` to the
`IOContext` used by the REPL -->
- [ ] #55458 <!-- Allow for generically extracting unannotated string
-->
- [ ] #55457 <!-- Make AnnotateChar equality consider annotations -->
- [ ] #55453 <!-- Privatise the annotations API, for StyledStrings -->
- [ ] #55355 <!-- relocation: account for trailing path separator in
depot paths -->
- [ ] #55220 <!-- `isfile_casesensitive` fixes on Windows -->
- [ ] #55169 <!-- `propertynames` for SVD respects private argument -->
- [ ] #54457 <!-- Make `String(::Memory)` copy -->
- [ ] #53957 <!-- tweak how filtering is done for what packages should
be precompiled -->
- [ ] #51479 <!-- prevent code loading from lookin in the versioned
environment when building Julia -->
- [ ] #50813 <!-- More doctests for Sockets and capitalization fix -->
- [ ] #50157 <!-- improve docs for `@inbounds` and
`Base.@propagate_inbounds` -->
- [ ] #41244 <!-- Fix shell `cd` error when working dir has been deleted
-->
  • Loading branch information
KristofferC authored Sep 17, 2024
2 parents 616e455 + 1c516dc commit 6cb5b1f
Show file tree
Hide file tree
Showing 38 changed files with 387 additions and 170 deletions.
5 changes: 5 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,11 @@ endif
cp -R -L $(JULIAHOME)/base/* $(DESTDIR)$(datarootdir)/julia/base
cp -R -L $(JULIAHOME)/test/* $(DESTDIR)$(datarootdir)/julia/test
cp -R -L $(build_datarootdir)/julia/* $(DESTDIR)$(datarootdir)/julia

# Set .jl sources as read-only to match package directories
find $(DESTDIR)$(datarootdir)/julia/base -type f -name \*.jl -exec chmod 0444 '{}' \;
find $(DESTDIR)$(datarootdir)/julia/test -type f -name \*.jl -exec chmod 0444 '{}' \;

# Copy documentation
cp -R -L $(BUILDROOT)/doc/_build/html $(DESTDIR)$(docdir)/
# Remove various files which should not be installed
Expand Down
2 changes: 1 addition & 1 deletion base/Base.jl
Original file line number Diff line number Diff line change
Expand Up @@ -603,7 +603,7 @@ function __init__()
init_active_project()
append!(empty!(_sysimage_modules), keys(loaded_modules))
empty!(explicit_loaded_modules)
@assert isempty(loaded_precompiles)
empty!(loaded_precompiles) # If we load a packageimage when building the image this might not be empty
for (mod, key) in module_keys
loaded_precompiles[key => module_build_id(mod)] = mod
end
Expand Down
7 changes: 4 additions & 3 deletions base/abstractarray.jl
Original file line number Diff line number Diff line change
Expand Up @@ -3504,24 +3504,25 @@ function push!(a::AbstractVector{T}, item) where T
itemT = item isa T ? item : convert(T, item)::T
new_length = length(a) + 1
resize!(a, new_length)
a[new_length] = itemT
a[end] = itemT
return a
end

# specialize and optimize the single argument case
function push!(a::AbstractVector{Any}, @nospecialize x)
new_length = length(a) + 1
resize!(a, new_length)
a[new_length] = x
a[end] = x
return a
end
function push!(a::AbstractVector{Any}, @nospecialize x...)
@_terminates_locally_meta
na = length(a)
nx = length(x)
resize!(a, na + nx)
e = lastindex(a) - nx
for i = 1:nx
a[na+i] = x[i]
a[e+i] = x[i]
end
return a
end
Expand Down
1 change: 1 addition & 0 deletions base/compiler/typelimits.jl
Original file line number Diff line number Diff line change
Expand Up @@ -815,6 +815,7 @@ end
typenames[i] = Any.name
simplify[i] = false
types[j] = widen
typenames[j] = ijname
break
end
end
Expand Down
145 changes: 99 additions & 46 deletions base/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,8 @@ package root.
To get the root directory of the package that implements the current module
the form `pkgdir(@__MODULE__)` can be used.
If an extension module is given, the root of the parent package is returned.
```julia-repl
julia> pkgdir(Foo)
"/path/to/Foo.jl"
Expand All @@ -524,7 +526,19 @@ function pkgdir(m::Module, paths::String...)
rootmodule = moduleroot(m)
path = pathof(rootmodule)
path === nothing && return nothing
return joinpath(dirname(dirname(path)), paths...)
original = path
path, base = splitdir(dirname(path))
if base == "src"
# package source in `../src/Foo.jl`
elseif base == "ext"
# extension source in `../ext/FooExt.jl`
elseif basename(path) == "ext"
# extension source in `../ext/FooExt/FooExt.jl`
path = dirname(path)
else
error("Unexpected path structure for module source: $original")
end
return joinpath(path, paths...)
end

function get_pkgversion_from_path(path)
Expand Down Expand Up @@ -1152,7 +1166,7 @@ const TIMING_IMPORTS = Threads.Atomic{Int}(0)
# these return either the array of modules loaded from the path / content given
# or an Exception that describes why it couldn't be loaded
# and it reconnects the Base.Docs.META
function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}, ignore_native::Union{Nothing,Bool}=nothing)
function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}, ignore_native::Union{Nothing,Bool}=nothing; register::Bool=true)
if isnothing(ignore_native)
if JLOptions().code_coverage == 0 && JLOptions().malloc_log == 0
ignore_native = false
Expand Down Expand Up @@ -1201,23 +1215,11 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No
for M in restored
M = M::Module
if parentmodule(M) === M && PkgId(M) == pkg
register && register_root_module(M)
if timing_imports
elapsed = round((time_ns() - t_before) / 1e6, digits = 1)
elapsed_time = time_ns() - t_before
comp_time, recomp_time = cumulative_compile_time_ns() .- t_comp_before
print(lpad(elapsed, 9), " ms ")
parentid = get(EXT_PRIMED, pkg, nothing)
if parentid !== nothing
print(parentid.name, "")
end
print(pkg.name)
if comp_time > 0
printstyled(" ", Ryu.writefixed(Float64(100 * comp_time / (elapsed * 1e6)), 2), "% compilation time", color = Base.info_color())
end
if recomp_time > 0
perc = Float64(100 * recomp_time / comp_time)
printstyled(" (", perc < 1 ? "<1" : Ryu.writefixed(perc, 0), "% recompilation)", color = Base.warn_color())
end
println()
print_time_imports_report(M, elapsed_time, comp_time, recomp_time)
end
return M
end
Expand All @@ -1229,6 +1231,73 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No
end
end

# printing functions for @time_imports
# note that the time inputs are UInt64 on all platforms. Give default values here so that we don't have
# confusing UInt64 types in generate_precompile.jl
function print_time_imports_report(
mod::Module,
elapsed_time::UInt64=UInt64(1),
comp_time::UInt64=UInt64(1),
recomp_time::UInt64=UInt64(1)
)
print(lpad(round(elapsed_time / 1e6, digits=1), 9), " ms ")
ext_parent = extension_parent_name(mod)
if ext_parent !== nothing
print(ext_parent::String, "")
end
print(string(mod))
if comp_time > 0
perc = Ryu.writefixed(Float64(100 * comp_time / (elapsed_time)), 2)
printstyled(" $perc% compilation time", color = Base.info_color())
end
if recomp_time > 0
perc = Float64(100 * recomp_time / comp_time)
perc_show = perc < 1 ? "<1" : Ryu.writefixed(perc, 0)
printstyled(" ($perc_show% recompilation)", color = Base.warn_color())
end
println()
end
function print_time_imports_report_init(
mod::Module, i::Int=1,
elapsed_time::UInt64=UInt64(1),
comp_time::UInt64=UInt64(1),
recomp_time::UInt64=UInt64(1)
)
connector = i > 1 ? "" : ""
printstyled(" $connector ", color = :light_black)
print("$(round(elapsed_time / 1e6, digits=1)) ms $mod.__init__() ")
if comp_time > 0
perc = Ryu.writefixed(Float64(100 * (comp_time) / elapsed_time), 2)
printstyled("$perc% compilation time", color = Base.info_color())
end
if recomp_time > 0
perc = Float64(100 * recomp_time / comp_time)
printstyled(" ($(perc < 1 ? "<1" : Ryu.writefixed(perc, 0))% recompilation)", color = Base.warn_color())
end
println()
end

# if M is an extension, return the string name of the parent. Otherwise return nothing
function extension_parent_name(M::Module)
rootmodule = moduleroot(M)
src_path = pathof(rootmodule)
src_path === nothing && return nothing
pkgdir_parts = splitpath(src_path)
ext_pos = findlast(==("ext"), pkgdir_parts)
if ext_pos !== nothing && ext_pos >= length(pkgdir_parts) - 2
parent_package_root = joinpath(pkgdir_parts[1:ext_pos-1]...)
parent_package_project_file = locate_project_file(parent_package_root)
if parent_package_project_file isa String
d = parsed_toml(parent_package_project_file)
name = get(d, "name", nothing)
if name !== nothing
return name
end
end
end
return nothing
end

function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
# This function is also used by PkgCacheInspector.jl
restored = sv[1]::Vector{Any}
Expand Down Expand Up @@ -1265,31 +1334,18 @@ function run_module_init(mod::Module, i::Int=1)
# `i` informs ordering for the `@time_imports` report formatting
if TIMING_IMPORTS[] == 0
ccall(:jl_init_restored_module, Cvoid, (Any,), mod)
else
if isdefined(mod, :__init__)
connector = i > 1 ? "" : ""
printstyled(" $connector ", color = :light_black)

elapsedtime = time_ns()
cumulative_compile_timing(true)
compile_elapsedtimes = cumulative_compile_time_ns()
elseif isdefined(mod, :__init__)
elapsed_time = time_ns()
cumulative_compile_timing(true)
compile_elapsedtimes = cumulative_compile_time_ns()

ccall(:jl_init_restored_module, Cvoid, (Any,), mod)
ccall(:jl_init_restored_module, Cvoid, (Any,), mod)

elapsedtime = (time_ns() - elapsedtime) / 1e6
cumulative_compile_timing(false);
comp_time, recomp_time = (cumulative_compile_time_ns() .- compile_elapsedtimes) ./ 1e6
elapsed_time = time_ns() - elapsed_time
cumulative_compile_timing(false);
comp_time, recomp_time = cumulative_compile_time_ns() .- compile_elapsedtimes

print("$(round(elapsedtime, digits=1)) ms $mod.__init__() ")
if comp_time > 0
printstyled(Ryu.writefixed(Float64(100 * comp_time / elapsedtime), 2), "% compilation time", color = Base.info_color())
end
if recomp_time > 0
perc = Float64(100 * recomp_time / comp_time)
printstyled(" ($(perc < 1 ? "<1" : Ryu.writefixed(perc, 0))% recompilation)", color = Base.warn_color())
end
println()
end
print_time_imports_report_init(mod, i, elapsed_time, comp_time, recomp_time)
end
end

Expand Down Expand Up @@ -1403,7 +1459,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}
triggers = triggers::Union{String, Vector{String}}
triggers isa String && (triggers = [triggers])
id = PkgId(uuid5(parent.uuid::UUID, ext), ext)
if id in keys(EXT_PRIMED) || haskey(Base.loaded_modules, id)
if haskey(EXT_PRIMED, id) || haskey(Base.loaded_modules, id)
continue # extension is already primed or loaded, don't add it again
end
EXT_PRIMED[id] = parent
Expand Down Expand Up @@ -1833,8 +1889,7 @@ function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union
depmods[i] = dep
end
# then load the file
loaded = _include_from_serialized(pkg, path, ocachepath, depmods, ignore_native)
loaded isa Module && register_root_module(loaded)
loaded = _include_from_serialized(pkg, path, ocachepath, depmods, ignore_native; register = true)
return loaded
end

Expand Down Expand Up @@ -1901,8 +1956,7 @@ end
if dep === nothing
try
set_pkgorigin_version_path(modkey, modpath)
dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps)
dep isa Module && stalecheck && register_root_module(dep)
dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps; register = stalecheck)
finally
end_loading(modkey, dep)
end
Expand All @@ -1918,9 +1972,8 @@ end
end
restored = get(loaded_precompiles, pkg => newbuild_id, nothing)
if !isa(restored, Module)
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps)
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps; register = stalecheck)
end
isa(restored, Module) && stalecheck && register_root_module(restored)
isa(restored, Module) && return restored
@debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
@label check_next_path
Expand Down
2 changes: 2 additions & 0 deletions base/multidimensional.jl
Original file line number Diff line number Diff line change
Expand Up @@ -693,6 +693,8 @@ using .IteratorsMD
end
@inline checkindex(::Type{Bool}, inds::Tuple, I::CartesianIndex) =
checkbounds_indices(Bool, inds, I.I)
@inline checkindex(::Type{Bool}, inds::Tuple, i::AbstractRange{<:CartesianIndex}) =
isempty(i) | (checkindex(Bool, inds, first(i)) & checkindex(Bool, inds, last(i)))

# Indexing into Array with mixtures of Integers and CartesianIndices is
# extremely performance-sensitive. While the abstract fallbacks support this,
Expand Down
5 changes: 3 additions & 2 deletions base/reducedim.jl
Original file line number Diff line number Diff line change
Expand Up @@ -261,8 +261,9 @@ function _mapreducedim!(f, op, R::AbstractArray, A::AbstractArrayOrBroadcasted)
# use mapreduce_impl, which is probably better tuned to achieve higher performance
nslices = div(length(A), lsiz)
ibase = first(LinearIndices(A))-1
for i = 1:nslices
@inbounds R[i] = op(R[i], mapreduce_impl(f, op, A, ibase+1, ibase+lsiz))
for i in eachindex(R)
r = op(@inbounds(R[i]), mapreduce_impl(f, op, A, ibase+1, ibase+lsiz))
@inbounds R[i] = r
ibase += lsiz
end
return R
Expand Down
2 changes: 1 addition & 1 deletion base/strings/annotated.jl
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ a vector of region–annotation tuples.
In accordance with the semantics documented in [`AnnotatedString`](@ref), the
order of annotations returned matches the order in which they were applied.
See also: `annotate!`.
See also: [`annotate!`](@ref).
"""
annotations(s::AnnotatedString) = s.annotations

Expand Down
3 changes: 2 additions & 1 deletion base/strings/io.jl
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,8 @@ function _join_preserve_annotations(iterator, args...)
# in nature, we extract an `AnnotatedString`, otherwise we just extract
# a plain `String` from `io`.
if isconcretetype(et) || !isempty(io.annotations)
read(seekstart(io), AnnotatedString{String})
seekstart(io)
read(io, AnnotatedString{String})
else
String(take!(io.io))
end
Expand Down
3 changes: 2 additions & 1 deletion base/strings/unicode.jl
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ module Unicode

import Base: show, ==, hash, string, Symbol, isless, length, eltype,
convert, isvalid, ismalformed, isoverlong, iterate,
AnnotatedString, AnnotatedChar, annotated_chartransform
AnnotatedString, AnnotatedChar, annotated_chartransform,
annotations

# whether codepoints are valid Unicode scalar values, i.e. 0-0xd7ff, 0xe000-0x10ffff

Expand Down
9 changes: 9 additions & 0 deletions contrib/generate_precompile.jl
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ precompile(Base.__require_prelocked, (Base.PkgId, Nothing))
precompile(Base._require, (Base.PkgId, Nothing))
precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int))
precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int))
precompile(Tuple{typeof(Base.Threads.atomic_add!), Base.Threads.Atomic{Int}, Int})
precompile(Tuple{typeof(Base.Threads.atomic_sub!), Base.Threads.Atomic{Int}, Int})
# Pkg loading
precompile(Tuple{typeof(Base.Filesystem.normpath), String, String, Vararg{String}})
Expand Down Expand Up @@ -163,6 +165,8 @@ for match = Base._methods(+, (Int, Int), -1, Base.get_world_counter())
push!(Expr[], Expr(:return, false))
vcat(String[], String[])
k, v = (:hello => nothing)
Base.print_time_imports_report(Base)
Base.print_time_imports_report_init(Base)
# Preferences uses these
get(Dict{String,Any}(), "missing", nothing)
Expand All @@ -174,6 +178,11 @@ for match = Base._methods(+, (Int, Int), -1, Base.get_world_counter())
# interactive statup uses this
write(IOBuffer(), "")
# not critical, but helps hide unrelated compilation from @time when using --trace-compile
foo() = rand(2,2) * rand(2,2)
@time foo()
@time foo()
break # only actually need to do this once
end
"""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
70878dd96911d6960537dfee2a820d98
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
87d2bdc6c85cbbce5302aab8ffe92fc542c9c71a396844fcc04c0416be059b00298b4816ab5e5491dbf865660a3a6152f1c245875a1ec75fb49b4c7ba0d303d8

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
248ce4c4b85394431e71b6f555f82216
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
84f193d72062199f2791ba7f7c2b6eb547e1a97569ae20cbb3557bc291715dd0cf55b2e121c4829b4572929e97255fd53b9210a168a7fde36f09ae6cf76c7f40
Loading

0 comments on commit 6cb5b1f

Please sign in to comment.