From 6abaed19db4a1a64b03af76981620da6a295c7e0 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 27 Feb 2024 08:57:49 -0500 Subject: [PATCH 01/55] add IR encoding for EnterNode (#53482) fixes #53248 (cherry picked from commit b3b27360672f87af78e90f43737e32988d9b8be1) --- src/ircode.c | 10 ++++++++++ src/serialize.h | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ircode.c b/src/ircode.c index 09f039db76424..8e2221ea06326 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -322,6 +322,11 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) jl_encode_value(s, jl_get_nth_field(v, 0)); jl_encode_value(s, jl_get_nth_field(v, 1)); } + else if (jl_is_enternode(v)) { + write_uint8(s->s, TAG_ENTERNODE); + jl_encode_value(s, jl_get_nth_field(v, 0)); + jl_encode_value(s, jl_get_nth_field(v, 1)); + } else if (jl_is_argument(v)) { write_uint8(s->s, TAG_ARGUMENT); jl_encode_value(s, jl_get_nth_field(v, 0)); @@ -722,6 +727,11 @@ static jl_value_t *jl_decode_value(jl_ircode_state *s) JL_GC_DISABLED set_nth_field(jl_gotoifnot_type, v, 0, jl_decode_value(s), 0); set_nth_field(jl_gotoifnot_type, v, 1, jl_decode_value(s), 0); return v; + case TAG_ENTERNODE: + v = jl_new_struct_uninit(jl_enternode_type); + set_nth_field(jl_enternode_type, v, 0, jl_decode_value(s), 0); + set_nth_field(jl_enternode_type, v, 1, jl_decode_value(s), 0); + return v; case TAG_ARGUMENT: v = jl_new_struct_uninit(jl_argument_type); set_nth_field(jl_argument_type, v, 0, jl_decode_value(s), 0); diff --git a/src/serialize.h b/src/serialize.h index 1bd29e9cc5911..2a91189dce739 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -65,8 +65,9 @@ extern "C" { #define TAG_RELOC_METHODROOT 57 #define TAG_BINDING 58 #define TAG_MEMORYT 59 +#define TAG_ENTERNODE 60 -#define LAST_TAG 59 +#define LAST_TAG 60 #define write_uint8(s, n) ios_putc((n), (s)) #define read_uint8(s) ((uint8_t)ios_getc((s))) From 79421de930a470f67a044ea6aa29edb9c233d430 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Wed, 28 Feb 2024 06:46:20 +0100 Subject: [PATCH 02/55] Avoid compiler warning about redefining jl_globalref_t (#53499) (cherry picked from commit 71f68b4ce9189e64f320631f3f74ffb3dd10e875) --- src/julia.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/julia.h b/src/julia.h index 852b671f744f6..6259da6416f3d 100644 --- a/src/julia.h +++ b/src/julia.h @@ -636,11 +636,11 @@ typedef struct _jl_module_t { intptr_t hash; } jl_module_t; -typedef struct _jl_globalref_t { +struct _jl_globalref_t { jl_module_t *mod; jl_sym_t *name; jl_binding_t *binding; -} jl_globalref_t; +}; // one Type-to-Value entry typedef struct _jl_typemap_entry_t { From 430114792d4ed2520c89a2a3dd8ca05efc7f61c5 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Thu, 29 Feb 2024 07:34:54 +0900 Subject: [PATCH 03/55] update staled `Core.Compiler.Effects` documentation (#53507) (cherry picked from commit c80a9647166d59782eff6be8c75a5052e17483d7) --- base/compiler/effects.jl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/base/compiler/effects.jl b/base/compiler/effects.jl index a864925b23eb1..a3d30baef9efa 100644 --- a/base/compiler/effects.jl +++ b/base/compiler/effects.jl @@ -43,15 +43,16 @@ following meanings: except that it may access or modify mutable memory pointed to by its call arguments. This may later be refined to `ALWAYS_TRUE` in a case when call arguments are known to be immutable. This state corresponds to LLVM's `inaccessiblemem_or_argmemonly` function attribute. -- `noub::Bool`: indicates that the method will not execute any undefined behavior (for any input). +- `noub::UInt8`: indicates that the method will not execute any undefined behavior (for any input). Note that undefined behavior may technically cause the method to violate any other effect assertions (such as `:consistent` or `:effect_free`) as well, but we do not model this, and they assume the absence of undefined behavior. + * `ALWAYS_TRUE`: this method is guaranteed to not execute any undefined behavior. + * `ALWAYS_FALSE`: this method may execute undefined behavior. + * `NOUB_IF_NOINBOUNDS`: this method is guaranteed to not execute any undefined behavior + if the caller does not set nor propagate the `@inbounds` context. - `nonoverlayed::Bool`: indicates that any methods that may be called within this method are not defined in an [overlayed method table](@ref OverlayMethodTable). -- `noinbounds::Bool`: If set, indicates that this method does not read the parent's `:inbounds` - state. In particular, it does not have any reached `:boundscheck` exprs, not propagates inbounds - to any children that do. Note that the representations above are just internal implementation details and thus likely to change in the future. See [`Base.@assume_effects`](@ref) for more detailed explanation @@ -92,9 +93,7 @@ The output represents the state of different effect properties in the following 7. `noub` (`u`): - `+u` (green): `true` - `-u` (red): `false` -8. `noinbounds` (`i`): - - `+i` (green): `true` - - `-i` (red): `false` + - `?u` (yellow): `NOUB_IF_NOINBOUNDS` Additionally, if the `nonoverlayed` property is false, a red prime symbol (′) is displayed after the tuple. """ From b701b5dcdf3e95c31ef00e388e21d6f85c1eeca2 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 29 Feb 2024 15:22:15 +0100 Subject: [PATCH 04/55] add back an alias for `check_top_bit` (#53523) Used in some packages (e.g. rfourquet/BitIntegers.jl) Renamed in https://github.com/JuliaLang/julia/pull/53166 (cherry picked from commit 24aaf00819e49ba5c377b9e9deed287e54333129) --- base/boot.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/boot.jl b/base/boot.jl index 8ca6d392ead67..f8ed86b0fb6b7 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -969,6 +969,7 @@ arrayset(inbounds::Bool, A::Array{T}, x::Any, i::Int...) where {T} = Main.Base.s arraysize(a::Array) = a.size arraysize(a::Array, i::Int) = sle_int(i, nfields(a.size)) ? getfield(a.size, i) : 1 export arrayref, arrayset, arraysize, const_arrayref +const check_top_bit = check_sign_bit # For convenience EnterNode(old::EnterNode, new_dest::Int) = isdefined(old, :scope) ? From 1366e3453fdee2b7d3121e78d05de7ac3356e17d Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 29 Feb 2024 18:04:39 -0500 Subject: [PATCH 05/55] add _readdirx for returning more object info gathered during dir scan (#53377) (cherry picked from commit 989c4db50e0ca59b890af2f1800004ecf91c0faf) --- base/file.jl | 85 ++++++++++++++++++++++++++++++++++++++++++++++++---- test/file.jl | 8 +++++ test/misc.jl | 5 ++-- 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/base/file.jl b/base/file.jl index 6347f042e3422..7f5caaaf96c6b 100644 --- a/base/file.jl +++ b/base/file.jl @@ -913,7 +913,79 @@ julia> readdir(abspath("base"), join=true) "/home/JuliaUser/dev/julia/base/weakkeydict.jl" ``` """ -function readdir(dir::AbstractString; join::Bool=false, sort::Bool=true) +readdir(; join::Bool=false, kwargs...) = readdir(join ? pwd() : "."; join, kwargs...)::Vector{String} +readdir(dir::AbstractString; kwargs...) = _readdir(dir; return_objects=false, kwargs...)::Vector{String} + +# this might be better as an Enum but they're not available here +# UV_DIRENT_T +const UV_DIRENT_UNKNOWN = Cint(0) +const UV_DIRENT_FILE = Cint(1) +const UV_DIRENT_DIR = Cint(2) +const UV_DIRENT_LINK = Cint(3) +const UV_DIRENT_FIFO = Cint(4) +const UV_DIRENT_SOCKET = Cint(5) +const UV_DIRENT_CHAR = Cint(6) +const UV_DIRENT_BLOCK = Cint(7) + +""" + DirEntry + +A type representing a filesystem entry that contains the name of the entry, the directory, and +the raw type of the entry. The full path of the entry can be obtained lazily by accessing the +`path` field. The type of the entry can be checked for by calling [`isfile`](@ref), [`isdir`](@ref), +[`islink`](@ref), [`isfifo`](@ref), [`issocket`](@ref), [`ischardev`](@ref), and [`isblockdev`](@ref) +""" +struct DirEntry + dir::String + name::String + rawtype::Cint +end +function Base.getproperty(obj::DirEntry, p::Symbol) + if p === :path + return joinpath(obj.dir, obj.name) + else + return getfield(obj, p) + end +end +Base.propertynames(::DirEntry) = (:dir, :name, :path, :rawtype) +Base.isless(a::DirEntry, b::DirEntry) = a.dir == b.dir ? isless(a.name, b.name) : isless(a.dir, b.dir) +Base.hash(o::DirEntry, h::UInt) = hash(o.dir, hash(o.name, hash(o.rawtype, h))) +Base.:(==)(a::DirEntry, b::DirEntry) = a.name == b.name && a.dir == b.dir && a.rawtype == b.rawtype +joinpath(obj::DirEntry, args...) = joinpath(obj.path, args...) +isunknown(obj::DirEntry) = obj.rawtype == UV_DIRENT_UNKNOWN +islink(obj::DirEntry) = isunknown(obj) ? islink(obj.path) : obj.rawtype == UV_DIRENT_LINK +isfile(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isfile(obj.path) : obj.rawtype == UV_DIRENT_FILE +isdir(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isdir(obj.path) : obj.rawtype == UV_DIRENT_DIR +isfifo(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isfifo(obj.path) : obj.rawtype == UV_DIRENT_FIFO +issocket(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? issocket(obj.path) : obj.rawtype == UV_DIRENT_SOCKET +ischardev(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? ischardev(obj.path) : obj.rawtype == UV_DIRENT_CHAR +isblockdev(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isblockdev(obj.path) : obj.rawtype == UV_DIRENT_BLOCK +realpath(obj::DirEntry) = realpath(obj.path) + +""" + _readdirx(dir::AbstractString=pwd(); sort::Bool = true) -> Vector{DirEntry} + +Return a vector of [`DirEntry`](@ref) objects representing the contents of the directory `dir`, +or the current working directory if not given. If `sort` is true, the returned vector is +sorted by name. + +Unlike [`readdir`](@ref), `_readdirx` returns [`DirEntry`](@ref) objects, which contain the name of the +file, the directory it is in, and the type of the file which is determined during the +directory scan. This means that calls to [`isfile`](@ref), [`isdir`](@ref), [`islink`](@ref), [`isfifo`](@ref), +[`issocket`](@ref), [`ischardev`](@ref), and [`isblockdev`](@ref) can be made on the +returned objects without further stat calls. However, for some filesystems, the type of the file +cannot be determined without a stat call. In these cases the `rawtype` field of the [`DirEntry`](@ref)) +object will be 0 (`UV_DIRENT_UNKNOWN`) and [`isfile`](@ref) etc. will fall back to a `stat` call. + +```julia +for obj in _readdirx() + isfile(obj) && println("\$(obj.name) is a file with path \$(obj.path)") +end +``` +""" +_readdirx(dir::AbstractString=pwd(); sort::Bool=true) = _readdir(dir; return_objects=true, sort)::Vector{DirEntry} + +function _readdir(dir::AbstractString; return_objects::Bool=false, join::Bool=false, sort::Bool=true) # Allocate space for uv_fs_t struct req = Libc.malloc(_sizeof_uv_fs) try @@ -923,11 +995,16 @@ function readdir(dir::AbstractString; join::Bool=false, sort::Bool=true) err < 0 && uv_error("readdir($(repr(dir)))", err) # iterate the listing into entries - entries = String[] + entries = return_objects ? DirEntry[] : String[] ent = Ref{uv_dirent_t}() while Base.UV_EOF != ccall(:uv_fs_scandir_next, Cint, (Ptr{Cvoid}, Ptr{uv_dirent_t}), req, ent) name = unsafe_string(ent[].name) - push!(entries, join ? joinpath(dir, name) : name) + if return_objects + rawtype = ent[].typ + push!(entries, DirEntry(dir, name, rawtype)) + else + push!(entries, join ? joinpath(dir, name) : name) + end end # Clean up the request string @@ -941,8 +1018,6 @@ function readdir(dir::AbstractString; join::Bool=false, sort::Bool=true) Libc.free(req) end end -readdir(; join::Bool=false, sort::Bool=true) = - readdir(join ? pwd() : ".", join=join, sort=sort) """ walkdir(dir; topdown=true, follow_symlinks=false, onerror=throw) diff --git a/test/file.jl b/test/file.jl index c32f655eb677e..a476895af442a 100644 --- a/test/file.jl +++ b/test/file.jl @@ -31,6 +31,8 @@ if !Sys.iswindows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER symlink(subdir, dirlink) @test stat(dirlink) == stat(subdir) @test readdir(dirlink) == readdir(subdir) + @test map(o->o.names, Base.Filesystem._readdirx(dirlink)) == map(o->o.names, Base.Filesystem._readdirx(subdir)) + @test realpath.(Base.Filesystem._readdirx(dirlink)) == realpath.(Base.Filesystem._readdirx(subdir)) # relative link relsubdirlink = joinpath(subdir, "rel_subdirlink") @@ -38,6 +40,7 @@ if !Sys.iswindows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER symlink(reldir, relsubdirlink) @test stat(relsubdirlink) == stat(subdir2) @test readdir(relsubdirlink) == readdir(subdir2) + @test Base.Filesystem._readdirx(relsubdirlink) == Base.Filesystem._readdirx(subdir2) # creation of symlink to directory that does not yet exist new_dir = joinpath(subdir, "new_dir") @@ -56,6 +59,7 @@ if !Sys.iswindows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER mkdir(new_dir) touch(foo_file) @test readdir(new_dir) == readdir(nedlink) + @test realpath.(Base.Filesystem._readdirx(new_dir)) == realpath.(Base.Filesystem._readdirx(nedlink)) rm(foo_file) rm(new_dir) @@ -1438,6 +1442,10 @@ rm(dirwalk, recursive=true) touch(randstring()) end @test issorted(readdir()) + @test issorted(Base.Filesystem._readdirx()) + @test map(o->o.name, Base.Filesystem._readdirx()) == readdir() + @test map(o->o.path, Base.Filesystem._readdirx()) == readdir(join=true) + @test count(isfile, readdir(join=true)) == count(isfile, Base.Filesystem._readdirx()) end end end diff --git a/test/misc.jl b/test/misc.jl index 6597ecf8a8428..e870c7f491c13 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1363,9 +1363,10 @@ end @test isdefined(KwdefWithEsc_TestModule, :Struct) @testset "exports of modules" begin - for (_, mod) in Base.loaded_modules + @testset "$mod" for (_, mod) in Base.loaded_modules mod === Main && continue # Main exports everything - for v in names(mod) + @testset "$v" for v in names(mod) + isdefined(mod, v) || @error "missing $v in $mod" @test isdefined(mod, v) end end From 0520b80a106e8e60c1f00234ed77dba196f74c32 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 29 Feb 2024 21:38:38 -0500 Subject: [PATCH 06/55] fix InteractiveUtils call in Base.runtests on failure (#53525) Noticed in CI that `Base.runtests(["fail"])` fails here instead of throwing the correct error later, since #53326. (cherry picked from commit 95f54c4dcb58e4f15f53f8a13a930db8e035ea89) --- base/util.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/base/util.jl b/base/util.jl index 847d89e895af4..c91f04e8baf96 100644 --- a/base/util.jl +++ b/base/util.jl @@ -694,11 +694,9 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), nothing catch buf = PipeBuffer() - original_load_path = copy(Base.LOAD_PATH); empty!(Base.LOAD_PATH); pushfirst!(Base.LOAD_PATH, "@stdlib") - let InteractiveUtils = Base.require_stdlib(Base, :InteractiveUtils) + let InteractiveUtils = Base.require_stdlib(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")) @invokelatest InteractiveUtils.versioninfo(buf) end - empty!(Base.LOAD_PATH); append!(Base.LOAD_PATH, original_load_path) error("A test has failed. Please submit a bug report (https://github.com/JuliaLang/julia/issues)\n" * "including error messages above and the output of versioninfo():\n$(read(buf, String))") end From 8912805e4d15580a84bf41db4d919b0c84bde129 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Fri, 16 Feb 2024 09:00:02 -0500 Subject: [PATCH 07/55] Add a lazy `logrange` function and `LogRange` type (#39071) (cherry picked from commit 3ed2b497910e091c80035339598e4c28d98b7188) --- NEWS.md | 1 + base/exports.jl | 2 + base/range.jl | 190 ++++++++++++++++++++++++++++++++++++++++- base/show.jl | 7 ++ base/twiceprecision.jl | 16 ++++ doc/src/base/math.md | 2 + test/ranges.jl | 83 ++++++++++++++++++ 7 files changed, 297 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index fc3a2304ea72a..42d36b606e839 100644 --- a/NEWS.md +++ b/NEWS.md @@ -83,6 +83,7 @@ New library functions * `in!(x, s::AbstractSet)` will return whether `x` is in `s`, and insert `x` in `s` if not. * The new `Libc.mkfifo` function wraps the `mkfifo` C function on Unix platforms ([#34587]). +* `logrange(start, stop; length)` makes a range of constant ratio, instead of constant step ([#39071]) * `copyuntil(out, io, delim)` and `copyline(out, io)` copy data into an `out::IO` stream ([#48273]). * `eachrsplit(string, pattern)` iterates split substrings right to left. * `Sys.username()` can be used to return the current user's username ([#51897]). diff --git a/base/exports.jl b/base/exports.jl index fbb5b449719d6..a31ea1b0fa842 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -412,6 +412,7 @@ export isperm, issorted, last, + logrange, mapslices, max, maximum!, @@ -1095,6 +1096,7 @@ public Generator, ImmutableDict, OneTo, + LogRange, AnnotatedString, AnnotatedChar, UUID, diff --git a/base/range.jl b/base/range.jl index 9480b46d8fcf1..4f92b305564cd 100644 --- a/base/range.jl +++ b/base/range.jl @@ -70,6 +70,7 @@ Valid invocations of range are: * Call `range` with one of `stop` or `length`. `start` and `step` will be assumed to be one. See Extended Help for additional details on the returned type. +See also [`logrange`](@ref) for logarithmically spaced points. # Examples ```jldoctest @@ -252,10 +253,13 @@ end ## 1-dimensional ranges ## """ - AbstractRange{T} + AbstractRange{T} <: AbstractVector{T} -Supertype for ranges with elements of type `T`. -[`UnitRange`](@ref) and other types are subtypes of this. +Supertype for linear ranges with elements of type `T`. +[`UnitRange`](@ref), [`LinRange`](@ref) and other types are subtypes of this. + +All subtypes must define [`step`](@ref). +Thus [`LogRange`](@ref Base.LogRange) is not a subtype of `AbstractRange`. """ abstract type AbstractRange{T} <: AbstractArray{T,1} end @@ -550,6 +554,8 @@ julia> collect(LinRange(-0.1, 0.3, 5)) 0.19999999999999998 0.3 ``` + +See also [`Logrange`](@ref Base.LogRange) for logarithmically spaced points. """ struct LinRange{T,L<:Integer} <: AbstractRange{T} start::T @@ -620,7 +626,7 @@ parameters `pre` and `post` characters for each printed row, `sep` separator string between printed elements, `hdots` string for the horizontal ellipsis. """ -function print_range(io::IO, r::AbstractRange, +function print_range(io::IO, r::AbstractArray, pre::AbstractString = " ", sep::AbstractString = ", ", post::AbstractString = "", @@ -1488,3 +1494,179 @@ julia> mod(3, 0:2) # mod(3, 3) """ mod(i::Integer, r::OneTo) = mod1(i, last(r)) mod(i::Integer, r::AbstractUnitRange{<:Integer}) = mod(i-first(r), length(r)) + first(r) + + +""" + logrange(start, stop, length) + logrange(start, stop; length) + +Construct a specialized array whose elements are spaced logarithmically +between the given endpoints. That is, the ratio of successive elements is +a constant, calculated from the length. + +This is similar to `geomspace` in Python. Unlike `PowerRange` in Mathematica, +you specify the number of elements not the ratio. +Unlike `logspace` in Python and Matlab, the `start` and `stop` arguments are +always the first and last elements of the result, not powers applied to some base. + +# Examples +```jldoctest +julia> logrange(10, 4000, length=3) +3-element Base.LogRange{Float64, Base.TwicePrecision{Float64}}: + 10.0, 200.0, 4000.0 + +julia> ans[2] ≈ sqrt(10 * 4000) # middle element is the geometric mean +true + +julia> range(10, 40, length=3)[2] ≈ (10 + 40)/2 # arithmetic mean +true + +julia> logrange(1f0, 32f0, 11) +11-element Base.LogRange{Float32, Float64}: + 1.0, 1.41421, 2.0, 2.82843, 4.0, 5.65685, 8.0, 11.3137, 16.0, 22.6274, 32.0 + +julia> logrange(1, 1000, length=4) ≈ 10 .^ (0:3) +true +``` + +See the [`LogRange`](@ref Base.LogRange) type for further details. + +See also [`range`](@ref) for linearly spaced points. + +!!! compat "Julia 1.11" + This function requires at least Julia 1.11. +""" +logrange(start::Real, stop::Real, length::Integer) = LogRange(start, stop, Int(length)) +logrange(start::Real, stop::Real; length::Integer) = logrange(start, stop, length) + + +""" + LogRange{T}(start, stop, len) <: AbstractVector{T} + +A range whose elements are spaced logarithmically between `start` and `stop`, +with spacing controlled by `len`. Returned by [`logrange`](@ref). + +Like [`LinRange`](@ref), the first and last elements will be exactly those +provided, but intermediate values may have small floating-point errors. +These are calculated using the logs of the endpoints, which are +stored on construction, often in higher precision than `T`. + +# Examples +```jldoctest +julia> logrange(1, 4, length=5) +5-element Base.LogRange{Float64, Base.TwicePrecision{Float64}}: + 1.0, 1.41421, 2.0, 2.82843, 4.0 + +julia> Base.LogRange{Float16}(1, 4, 5) +5-element Base.LogRange{Float16, Float64}: + 1.0, 1.414, 2.0, 2.828, 4.0 + +julia> logrange(1e-310, 1e-300, 11)[1:2:end] +6-element Vector{Float64}: + 1.0e-310 + 9.999999999999974e-309 + 9.999999999999981e-307 + 9.999999999999988e-305 + 9.999999999999994e-303 + 1.0e-300 + +julia> prevfloat(1e-308, 5) == ans[2] +true +``` + +Note that integer eltype `T` is not allowed. +Use for instance `round.(Int, xs)`, or explicit powers of some integer base: + +```jldoctest +julia> xs = logrange(1, 512, 4) +4-element Base.LogRange{Float64, Base.TwicePrecision{Float64}}: + 1.0, 8.0, 64.0, 512.0 + +julia> 2 .^ (0:3:9) |> println +[1, 8, 64, 512] +``` + +!!! compat "Julia 1.11" + This type requires at least Julia 1.11. +""" +struct LogRange{T<:Real,X} <: AbstractArray{T,1} + start::T + stop::T + len::Int + extra::Tuple{X,X} + function LogRange{T}(start::T, stop::T, len::Int) where {T<:Real} + if T <: Integer + # LogRange{Int}(1, 512, 4) produces InexactError: Int64(7.999999999999998) + throw(ArgumentError("LogRange{T} does not support integer types")) + end + if iszero(start) || iszero(stop) + throw(DomainError((start, stop), + "LogRange cannot start or stop at zero")) + elseif start < 0 || stop < 0 + # log would throw, but _log_twice64_unchecked does not + throw(DomainError((start, stop), + "LogRange does not accept negative numbers")) + elseif !isfinite(start) || !isfinite(stop) + throw(DomainError((start, stop), + "LogRange is only defined for finite start & stop")) + elseif len < 0 + throw(ArgumentError(LazyString( + "LogRange(", start, ", ", stop, ", ", len, "): can't have negative length"))) + elseif len == 1 && start != stop + throw(ArgumentError(LazyString( + "LogRange(", start, ", ", stop, ", ", len, "): endpoints differ, while length is 1"))) + end + ex = _logrange_extra(start, stop, len) + new{T,typeof(ex[1])}(start, stop, len, ex) + end +end + +function LogRange{T}(start::Real, stop::Real, len::Integer) where {T} + LogRange{T}(convert(T, start), convert(T, stop), convert(Int, len)) +end +function LogRange(start::Real, stop::Real, len::Integer) + T = float(promote_type(typeof(start), typeof(stop))) + LogRange{T}(convert(T, start), convert(T, stop), convert(Int, len)) +end + +size(r::LogRange) = (r.len,) +length(r::LogRange) = r.len + +first(r::LogRange) = r.start +last(r::LogRange) = r.stop + +function _logrange_extra(a::Real, b::Real, len::Int) + loga = log(1.0 * a) # widen to at least Float64 + logb = log(1.0 * b) + (loga/(len-1), logb/(len-1)) +end +function _logrange_extra(a::Float64, b::Float64, len::Int) + loga = _log_twice64_unchecked(a) + logb = _log_twice64_unchecked(b) + # The reason not to do linear interpolation on log(a)..log(b) in `getindex` is + # that division of TwicePrecision is quite slow, so do it once on construction: + (loga/(len-1), logb/(len-1)) +end + +function getindex(r::LogRange{T}, i::Int) where {T} + @inline + @boundscheck checkbounds(r, i) + i == 1 && return r.start + i == r.len && return r.stop + # Main path uses Math.exp_impl for TwicePrecision, but is not perfectly + # accurate, hence the special cases for endpoints above. + logx = (r.len-i) * r.extra[1] + (i-1) * r.extra[2] + x = _exp_allowing_twice64(logx) + return T(x) +end + +function show(io::IO, r::LogRange{T}) where {T} + print(io, "LogRange{", T, "}(") + ioc = IOContext(io, :typeinfo => T) + show(ioc, first(r)) + print(io, ", ") + show(ioc, last(r)) + print(io, ", ") + show(io, length(r)) + print(io, ')') +end diff --git a/base/show.jl b/base/show.jl index a5bf5f73ba0ed..217b22ab39ef8 100644 --- a/base/show.jl +++ b/base/show.jl @@ -23,6 +23,13 @@ function show(io::IO, ::MIME"text/plain", r::LinRange) print_range(io, r) end +function show(io::IO, ::MIME"text/plain", r::LogRange) # display LogRange like LinRange + isempty(r) && return show(io, r) + summary(io, r) + println(io, ":") + print_range(io, r, " ", ", ", "", " \u2026 ") +end + function _isself(ft::DataType) ftname = ft.name isdefined(ftname, :mt) || return false diff --git a/base/twiceprecision.jl b/base/twiceprecision.jl index 955bfc97b16ff..0de6270cafb2d 100644 --- a/base/twiceprecision.jl +++ b/base/twiceprecision.jl @@ -785,3 +785,19 @@ _tp_prod(t::TwicePrecision) = t x.hi < y.hi || ((x.hi == y.hi) & (x.lo < y.lo)) isbetween(a, x, b) = a <= x <= b || b <= x <= a + +# These functions exist for use in LogRange: + +_exp_allowing_twice64(x::Number) = exp(x) +_exp_allowing_twice64(x::TwicePrecision{Float64}) = Math.exp_impl(x.hi, x.lo, Val(:ℯ)) + +# No error on negative x, and for NaN/Inf this returns junk: +function _log_twice64_unchecked(x::Float64) + xu = reinterpret(UInt64, x) + if xu < (UInt64(1)<<52) # x is subnormal + xu = reinterpret(UInt64, x * 0x1p52) # normalize x + xu &= ~sign_mask(Float64) + xu -= UInt64(52) << 52 # mess with the exponent + end + TwicePrecision(Math._log_ext(xu)...) +end diff --git a/doc/src/base/math.md b/doc/src/base/math.md index 86a7e4d8d3173..c1b18c0b66d86 100644 --- a/doc/src/base/math.md +++ b/doc/src/base/math.md @@ -39,6 +39,8 @@ Base.:(:) Base.range Base.OneTo Base.StepRangeLen +Base.logrange +Base.LogRange Base.:(==) Base.:(!=) Base.:(!==) diff --git a/test/ranges.jl b/test/ranges.jl index 549078bab45fe..4660a96dfc16a 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -2603,3 +2603,86 @@ end errmsg = ("deliberately unsupported for CartesianIndex", "StepRangeLen") @test_throws errmsg range(CartesianIndex(1), step=CartesianIndex(1), length=3) end + +@testset "logrange" begin + # basic idea + @test logrange(2, 16, 4) ≈ [2, 4, 8, 16] + @test logrange(1/8, 8.0, 7) ≈ [0.125, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0] + @test logrange(1000, 1, 4) ≈ [1000, 100, 10, 1] + @test logrange(1, 10^9, 19)[1:2:end] ≈ 10 .^ (0:9) + + # endpoints + @test logrange(0.1f0, 100, 33)[1] === 0.1f0 + @test logrange(0.789, 123_456, 135_790)[[begin, end]] == [0.789, 123_456] + @test logrange(nextfloat(0f0), floatmax(Float32), typemax(Int))[end] === floatmax(Float32) + @test logrange(nextfloat(Float16(0)), floatmax(Float16), 66_000)[end] === floatmax(Float16) + @test first(logrange(pi, 2pi, 3000)) === logrange(pi, 2pi, 3000)[1] === Float64(pi) + if Int == Int64 + @test logrange(0.1, 1000, 2^54)[end] === 1000.0 + end + + # empty, only, constant + @test first(logrange(1, 2, 0)) === 1.0 + @test last(logrange(1, 2, 0)) === 2.0 + @test collect(logrange(1, 2, 0)) == Float64[] + @test only(logrange(2pi, 2pi, 1)) === logrange(2pi, 2pi, 1)[1] === 2pi + @test logrange(1, 1, 3) == fill(1.0, 3) + + # subnormal Float64 + x = logrange(1e-320, 1e-300, 21) .* 1e300 + @test x ≈ logrange(1e-20, 1, 21) rtol=1e-6 + + # types + @test eltype(logrange(1, 10, 3)) == Float64 + @test eltype(logrange(1, 10, Int32(3))) == Float64 + @test eltype(logrange(1, 10f0, 3)) == Float32 + @test eltype(logrange(1f0, 10, 3)) == Float32 + @test eltype(logrange(1, big(10), 3)) == BigFloat + @test logrange(big"0.3", big(pi), 50)[1] == big"0.3" + @test logrange(big"0.3", big(pi), 50)[end] == big(pi) + + # more constructors + @test logrange(1,2,length=3) === Base.LogRange(1,2,3) == Base.LogRange{Float64}(1,2,3) + @test logrange(1f0, 2f0, length=3) == Base.LogRange{Float32}(1,2,3) + + # errors + @test_throws UndefKeywordError logrange(1, 10) # no default length + @test_throws ArgumentError logrange(1, 10, -1) # negative length + @test_throws ArgumentError logrange(1, 10, 1) # endpoints must not differ + @test_throws DomainError logrange(1, -1, 3) # needs complex numbers + @test_throws DomainError logrange(-1, -2, 3) # not supported, for now + @test_throws MethodError logrange(1, 2+3im, length=4) # not supported, for now + @test_throws ArgumentError logrange(1, 10, 2)[true] # bad index + @test_throws BoundsError logrange(1, 10, 2)[3] + @test_throws ArgumentError Base.LogRange{Int}(1,4,5) # no integer ranges + @test_throws MethodError Base.LogRange(1,4, length=5) # type does not take keyword + # (not sure if these should ideally be DomainError or ArgumentError) + @test_throws DomainError logrange(1, Inf, 3) + @test_throws DomainError logrange(0, 2, 3) + @test_throws DomainError logrange(1, NaN, 3) + @test_throws DomainError logrange(NaN, 2, 3) + + # printing + @test repr(Base.LogRange(1,2,3)) == "LogRange{Float64}(1.0, 2.0, 3)" # like 2-arg show + @test repr("text/plain", Base.LogRange(1,2,3)) == "3-element Base.LogRange{Float64, Base.TwicePrecision{Float64}}:\n 1.0, 1.41421, 2.0" + @test repr("text/plain", Base.LogRange(1,2,0)) == "LogRange{Float64}(1.0, 2.0, 0)" # empty case +end + +@testset "_log_twice64_unchecked" begin + # it roughly works + @test big(Base._log_twice64_unchecked(exp(1))) ≈ 1.0 + @test big(Base._log_twice64_unchecked(exp(123))) ≈ 123.0 + + # it gets high accuracy + @test abs(big(log(4.0)) - log(big(4.0))) < 1e-16 + @test abs(big(Base._log_twice64_unchecked(4.0)) - log(big(4.0))) < 1e-30 + + # it handles subnormals + @test abs(big(Base._log_twice64_unchecked(1e-310)) - log(big(1e-310))) < 1e-20 + + # it accepts negative, NaN, etc without complaint: + @test Base._log_twice64_unchecked(-0.0).lo isa Float64 + @test Base._log_twice64_unchecked(-1.23).lo isa Float64 + @test Base._log_twice64_unchecked(NaN).lo isa Float64 + @test Base._log_twice64_unchecked(Inf).lo isa Float64 +end From 42a71a5c594c01aba5b1ce9c11d3b5025239e1da Mon Sep 17 00:00:00 2001 From: tecosaur Date: Sun, 18 Feb 2024 03:01:55 +0800 Subject: [PATCH 08/55] Allow AnnotatedStrings in log messages (#51802) Permitting annotated strings allows for styling information to be preserved through to log printing. (cherry picked from commit 199851801fa183c1bfac9b3ba2282655e2629b7f) --- stdlib/Logging/src/ConsoleLogger.jl | 13 +++++++++++-- stdlib/Logging/test/runtests.jl | 9 +++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/stdlib/Logging/src/ConsoleLogger.jl b/stdlib/Logging/src/ConsoleLogger.jl index ef901ebbbbc5d..2950ad1b8cf95 100644 --- a/stdlib/Logging/src/ConsoleLogger.jl +++ b/stdlib/Logging/src/ConsoleLogger.jl @@ -118,8 +118,17 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module end # Generate a text representation of the message and all key value pairs, - # split into lines. - msglines = [(indent=0, msg=l) for l in split(chomp(convert(String, string(message))::String), '\n')] + # split into lines. This is specialised to improve type inference, + # and reduce the risk of resulting method invalidations. + message = string(message) + msglines = if Base._isannotated(message) && !isempty(Base.annotations(message)) + message = Base.AnnotatedString(String(message), Base.annotations(message)) + @NamedTuple{indent::Int, msg::Union{SubString{Base.AnnotatedString{String}}, SubString{String}}}[ + (indent=0, msg=l) for l in split(chomp(message), '\n')] + else + [(indent=0, msg=l) for l in split( + chomp(convert(String, message)::String), '\n')] + end stream::IO = logger.stream if !(isopen(stream)::Bool) stream = stderr diff --git a/stdlib/Logging/test/runtests.jl b/stdlib/Logging/test/runtests.jl index 1a86c884295b4..57866ff0a18d2 100644 --- a/stdlib/Logging/test/runtests.jl +++ b/stdlib/Logging/test/runtests.jl @@ -52,6 +52,15 @@ end end @test String(take!(buf)) == "" + # Check that the AnnotatedString path works too + with_logger(logger) do + @info Base.AnnotatedString("test") + end + @test String(take!(buf)) == + """ + [ Info: test + """ + @testset "Default metadata formatting" begin @test Logging.default_metafmt(Logging.Debug, Base, :g, :i, expanduser("~/somefile.jl"), 42) == (:log_debug, "Debug:", "@ Base ~/somefile.jl:42") From 6904351d619df21b43194e1c3c8672c809ca5cf9 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Fri, 23 Feb 2024 17:04:29 -0500 Subject: [PATCH 09/55] improve `--heap-size-hint` arg handling (#48050) Previously, `--heap-size-hint` would silently ignore many flavors of "bad" input, parsing things like "3PB" as 3 bytes. This change makes it significantly less permissive, erroring unless it can parse a number (still relying on the C `sscanf` `%Lf` format specifier there) with an optional unit (case-insensitive, either with or without the trailing `b`). Also test it. (cherry picked from commit 138aba7320c4c37b1ad6330ec46531077f8ee516) --- doc/man/julia.1 | 7 +- doc/src/manual/command-line-interface.md | 2 +- src/jloptions.c | 90 ++++++++++++++---------- test/cmdlineargs.jl | 16 +++++ 4 files changed, 73 insertions(+), 42 deletions(-) diff --git a/doc/man/julia.1 b/doc/man/julia.1 index c58b00120b7ec..afde0326e4952 100644 --- a/doc/man/julia.1 +++ b/doc/man/julia.1 @@ -228,9 +228,10 @@ fallbacks to the latest compatible BugReporting.jl if not. For more information, --bug-report=help. .TP ---heap-size-hint= -Forces garbage collection if memory usage is higher than that value. The value can be -specified in units of K, M, G, T, or % of physical memory. +--heap-size-hint= +Forces garbage collection if memory usage is higher than the given value. +The value may be specified as a number of bytes, optionally in units of +KB, MB, GB, or TB, or as a percentage of physical memory with %. .TP --compile={yes*|no|all|min} diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 4faa3e8d3fc05..448964bf1ef8a 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -213,7 +213,7 @@ The following is a complete list of command-line switches available when launchi |`--output-incremental={yes\|no*}` |Generate an incremental output file (rather than complete)| |`--trace-compile={stderr,name}` |Print precompile statements for methods compiled during execution or save to a path| |`--image-codegen` |Force generate code in imaging mode| -|`--heap-size-hint=` |Forces garbage collection if memory usage is higher than that value. The memory hint might be specified in megabytes (e.g., 500M) or gigabytes (e.g., 1G)| +|`--heap-size-hint=` |Forces garbage collection if memory usage is higher than the given value. The value may be specified as a number of bytes, optionally in units of KB, MB, GB, or TB, or as a percentage of physical memory with %.| !!! compat "Julia 1.1" diff --git a/src/jloptions.c b/src/jloptions.c index 4b3f7209dc56d..570d021351104 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -8,6 +8,7 @@ #include #include + #include "julia_assert.h" #ifdef _OS_WINDOWS_ @@ -18,6 +19,15 @@ char *shlib_ext = ".dylib"; char *shlib_ext = ".so"; #endif +/* This simple hand-crafted tolower exists to avoid locale-dependent effects in + * behaviors (and utf8proc_tolower wasn't linking properly on all platforms) */ +static char ascii_tolower(char c) +{ + if ('A' <= c && c <= 'Z') + return c - 'A' + 'a'; + return c; +} + static const char system_image_path[256] = "\0" JL_SYSTEM_IMAGE_PATH; JL_DLLEXPORT const char *jl_get_default_sysimg_path(void) { @@ -187,9 +197,9 @@ static const char opts[] = " expressions. It first tries to use BugReporting.jl installed in current environment and\n" " fallbacks to the latest compatible BugReporting.jl if not. For more information, see\n" " --bug-report=help.\n\n" - - " --heap-size-hint= Forces garbage collection if memory usage is higher than that value.\n" - " The value can be specified in units of K, M, G, T, or % of physical memory.\n\n" + " --heap-size-hint= Forces garbage collection if memory usage is higher than the given value.\n" + " The value may be specified as a number of bytes, optionally in units of\n" + " KB, MB, GB, or TB, or as a percentage of physical memory with %.\n\n" ; static const char opts_hidden[] = @@ -801,43 +811,47 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) break; case opt_heap_size_hint: if (optarg != NULL) { - size_t endof = strlen(optarg); long double value = 0.0; - if (sscanf(optarg, "%Lf", &value) == 1 && value > 1e-7) { - char unit = optarg[endof - 1]; - uint64_t multiplier = 1ull; - switch (unit) { - case 'k': - case 'K': - multiplier <<= 10; - break; - case 'm': - case 'M': - multiplier <<= 20; - break; - case 'g': - case 'G': - multiplier <<= 30; - break; - case 't': - case 'T': - multiplier <<= 40; - break; - case '%': - if (value > 100) - jl_errorf("julia: invalid percentage specified in --heap-size-hint"); - uint64_t mem = uv_get_total_memory(); - uint64_t cmem = uv_get_constrained_memory(); - if (cmem > 0 && cmem < mem) - mem = cmem; - multiplier = mem/100; - break; - default: - jl_errorf("julia: invalid unit specified in --heap-size-hint"); - break; - } - jl_options.heap_size_hint = (uint64_t)(value * multiplier); + char unit[4] = {0}; + int nparsed = sscanf(optarg, "%Lf%3s", &value, unit); + if (nparsed == 0 || strlen(unit) > 2 || (strlen(unit) == 2 && ascii_tolower(unit[1]) != 'b')) { + jl_errorf("julia: invalid argument to --heap-size-hint (%s)", optarg); + } + uint64_t multiplier = 1ull; + switch (ascii_tolower(unit[0])) { + case '\0': + case 'b': + break; + case 'k': + multiplier <<= 10; + break; + case 'm': + multiplier <<= 20; + break; + case 'g': + multiplier <<= 30; + break; + case 't': + multiplier <<= 40; + break; + case '%': + if (value > 100) + jl_errorf("julia: invalid percentage specified in --heap-size-hint"); + uint64_t mem = uv_get_total_memory(); + uint64_t cmem = uv_get_constrained_memory(); + if (cmem > 0 && cmem < mem) + mem = cmem; + multiplier = mem/100; + break; + default: + jl_errorf("julia: invalid argument to --heap-size-hint (%s)", optarg); + break; + } + long double sz = value * multiplier; + if (isnan(sz) || sz < 0) { + jl_errorf("julia: invalid argument to --heap-size-hint (%s)", optarg); } + jl_options.heap_size_hint = sz < UINT64_MAX ? (uint64_t)sz : UINT64_MAX; } if (jl_options.heap_size_hint == 0) jl_errorf("julia: invalid memory size specified in --heap-size-hint"); diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 3af31965e63fc..498254bb26ce0 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -1148,3 +1148,19 @@ if Sys.islinux() && Sys.ARCH in (:i686, :x86_64) # rr is only available on these "_RR_TRACE_DIR" => temp_trace_dir); #=stderr, stdout=#)) end end + +@testset "--heap-size-hint" begin + exename = `$(Base.julia_cmd())` + @test errors_not_signals(`$exename --heap-size-hint -e "exit(0)"`) + @testset "--heap-size-hint=$str" for str in ["asdf","","0","1.2vb","b","GB","2.5GB̂","1.2gb2","42gigabytes","5gig","2GiB","NaNt"] + @test errors_not_signals(`$exename --heap-size-hint=$str -e "exit(0)"`) + end + k = 1024 + m = 1024k + g = 1024m + t = 1024g + @testset "--heap-size-hint=$str" for (str, val) in [("1", 1), ("1e7", 1e7), ("2.5e7", 2.5e7), ("1MB", 1m), ("2.5g", 2.5g), ("1e4kB", 1e4k), + ("1e100", typemax(UInt64)), ("1e500g", typemax(UInt64)), ("1e-12t", 1), ("500000000b", 500000000)] + @test parse(UInt64,read(`$exename --heap-size-hint=$str -E "Base.JLOptions().heap_size_hint"`, String)) == val + end +end From 7859e32979b82571d584a1b02bac82826f4a0042 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 1 Mar 2024 18:03:41 -0500 Subject: [PATCH 10/55] use more efficient `_readdirx` for tab completion (#53540) Fixes https://github.com/JuliaLang/julia/issues/53153 (cherry picked from commit 188e386e090fb83e89fb527d68fd23afee94c3b2) --- stdlib/REPL/src/REPLCompletions.jl | 35 +++++++++++++++--------------- stdlib/REPL/src/docview.jl | 8 ++++--- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 621cd6f9021fb..f5ef8e6bff53c 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -8,6 +8,7 @@ using Core: CodeInfo, MethodInstance, CodeInstance, Const const CC = Core.Compiler using Base.Meta using Base: propertynames, something, IdSet +using Base.Filesystem: _readdirx abstract type Completion end @@ -317,8 +318,8 @@ function cache_PATH() continue end - filesinpath = try - readdir(pathdir) + path_entries = try + _readdirx(pathdir) catch e # Bash allows dirs in PATH that can't be read, so we should as well. if isa(e, Base.IOError) || isa(e, Base.ArgumentError) @@ -328,13 +329,13 @@ function cache_PATH() rethrow() end end - for file in filesinpath + for entry in path_entries # In a perfect world, we would filter on whether the file is executable # here, or even on whether the current user can execute the file in question. try - if isfile(joinpath(pathdir, file)) - @lock PATH_cache_lock push!(PATH_cache, file) - push!(this_PATH_cache, file) + if isfile(entry) + @lock PATH_cache_lock push!(PATH_cache, entry.name) + push!(this_PATH_cache, entry.name) end catch e # `isfile()` can throw in rare cases such as when probing a @@ -378,11 +379,11 @@ function complete_path(path::AbstractString; else dir, prefix = splitdir(path) end - files = try + entries = try if isempty(dir) - readdir() + _readdirx() elseif isdir(dir) - readdir(dir) + _readdirx(dir) else return Completion[], dir, false end @@ -392,11 +393,10 @@ function complete_path(path::AbstractString; end matches = Set{String}() - for file in files - if startswith(file, prefix) - p = joinpath(dir, file) - is_dir = try isdir(p) catch ex; ex isa Base.IOError ? false : rethrow() end - push!(matches, is_dir ? file * "/" : file) + for entry in entries + if startswith(entry.name, prefix) + is_dir = try isdir(entry) catch ex; ex isa Base.IOError ? false : rethrow() end + push!(matches, is_dir ? entry.name * "/" : entry.name) end end @@ -1349,14 +1349,15 @@ function completions(string::String, pos::Int, context_module::Module=Main, shif append!(suggestions, project_deps_get_completion_candidates(s, dir)) end isdir(dir) || continue - for pname in readdir(dir) + for entry in _readdirx(dir) + pname = entry.name if pname[1] != '.' && pname != "METADATA" && pname != "REQUIRE" && startswith(pname, s) # Valid file paths are # .jl # /src/.jl # .jl/src/.jl - if isfile(joinpath(dir, pname)) + if isfile(entry) endswith(pname, ".jl") && push!(suggestions, PackageCompletion(pname[1:prevind(pname, end-2)])) else @@ -1365,7 +1366,7 @@ function completions(string::String, pos::Int, context_module::Module=Main, shif else pname end - if isfile(joinpath(dir, pname, "src", + if isfile(joinpath(entry, "src", "$mod_name.jl")) push!(suggestions, PackageCompletion(mod_name)) end diff --git a/stdlib/REPL/src/docview.jl b/stdlib/REPL/src/docview.jl index 79cc6b48d2f4d..feeeecbd97165 100644 --- a/stdlib/REPL/src/docview.jl +++ b/stdlib/REPL/src/docview.jl @@ -11,6 +11,8 @@ import Base.Docs: doc, formatdoc, parsedoc, apropos using Base: with_output_color, mapany, isdeprecated, isexported +using Base.Filesystem: _readdirx + import REPL using InteractiveUtils: subtypes @@ -379,9 +381,9 @@ function find_readme(m::Module)::Union{String, Nothing} path = dirname(mpath) top_path = pkgdir(m) while true - for file in readdir(path; join=true, sort=true) - isfile(file) && (basename(lowercase(file)) in ["readme.md", "readme"]) || continue - return file + for entry in _readdirx(path; sort=true) + isfile(entry) && (lowercase(entry.name) in ["readme.md", "readme"]) || continue + return entry.path end path == top_path && break # go no further than pkgdir path = dirname(path) # work up through nested modules From b63e08623e45658cd2c8c8cce04b9da804e825ca Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 1 Mar 2024 18:04:35 -0500 Subject: [PATCH 11/55] use `_readdirx` for `walkdir` (#53545) On a M2 Mac there is some benefit, but assumed to be much greater on slower filesystems. ``` # master julia> @btime collect(walkdir(expanduser("~/Downloads"))); 380.086 ms (310696 allocations: 25.29 MiB) # This PR julia> @btime collect(walkdir(expanduser("~/Downloads"))); 289.747 ms (103300 allocations: 7.50 MiB) ``` The implementations appear to produce the same result ``` julia> collect(walkdir(expanduser("~/Downloads"))) == collect(walkdirx(expanduser("~/Downloads"))) true ``` (cherry picked from commit 2b9595601e6e81b0c81376d7942497af22e222dd) --- base/file.jl | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/base/file.jl b/base/file.jl index 7f5caaaf96c6b..cb69cc5e0ee44 100644 --- a/base/file.jl +++ b/base/file.jl @@ -1073,18 +1073,16 @@ function walkdir(root; topdown=true, follow_symlinks=false, onerror=throw) end return end - content = tryf(readdir, root) - content === nothing && return - dirs = Vector{eltype(content)}() - files = Vector{eltype(content)}() - for name in content - path = joinpath(root, name) - + entries = tryf(_readdirx, root) + entries === nothing && return + dirs = Vector{String}() + files = Vector{String}() + for entry in entries # If we're not following symlinks, then treat all symlinks as files - if (!follow_symlinks && something(tryf(islink, path), true)) || !something(tryf(isdir, path), false) - push!(files, name) + if (!follow_symlinks && something(tryf(islink, entry), true)) || !something(tryf(isdir, entry), false) + push!(files, entry.name) else - push!(dirs, name) + push!(dirs, entry.name) end end From c01ef981d3843b50639e24bc3716ccb0fc18209d Mon Sep 17 00:00:00 2001 From: erich-9 Date: Sun, 3 Mar 2024 04:14:28 +0100 Subject: [PATCH 12/55] Always return a value in 1-d circshift! of abstractarray.jl (#53554) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sukera <11753998+Seelengrab@users.noreply.github.com> Co-authored-by: Mosè Giordano (cherry picked from commit 7179050c81cfcaca75419f4948bf59a025b50469) --- base/abstractarray.jl | 4 ++-- test/arrayops.jl | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 59ba7ad82c7f0..c9307c8c0f70f 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -3643,9 +3643,9 @@ end ## 1-d circshift ## function circshift!(a::AbstractVector, shift::Integer) n = length(a) - n == 0 && return + n == 0 && return a shift = mod(shift, n) - shift == 0 && return + shift == 0 && return a l = lastindex(a) reverse!(a, firstindex(a), l-shift) reverse!(a, l-shift+1, lastindex(a)) diff --git a/test/arrayops.jl b/test/arrayops.jl index a07e631e639f9..566dd44b8dcd9 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -797,6 +797,14 @@ end oa = OffsetVector(copy(a), -1) @test circshift!(oa, 1) === oa @test oa == circshift(OffsetVector(a, -1), 1) + + # 1d circshift! (#53554) + a = [] + @test circshift!(a, 1) === a + @test circshift!(a, 1) == [] + a = [1:5;] + @test circshift!(a, 10) === a + @test circshift!(a, 10) == 1:5 end @testset "circcopy" begin From 53180e48ad973e5fa479327fb22ff2e1e5dba284 Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:16:28 -0300 Subject: [PATCH 13/55] yet more atomics & cache-line fixes on work-stealing queue (#53424) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This regression at a large number of GC threads still shows up on a large aarch64 machine (80-core ARM Neoverse N1). ``` bench = "many_refs.jl" (gcthreads, threads) = (1, 1) [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback ┌─────────┬────────────┬─────────┬───────────┬────────────┬──────────────┬───────────────────┬──────────┬────────────┐ │ │ total time │ gc time │ mark time │ sweep time │ max GC pause │ time to safepoint │ max heap │ percent gc │ │ │ ms │ ms │ ms │ ms │ ms │ us │ MB │ % │ ├─────────┼────────────┼─────────┼───────────┼────────────┼──────────────┼───────────────────┼──────────┼────────────┤ │ minimum │ 3807 │ 2826 │ 2633 │ 193 │ 960 │ 13 │ 892 │ 74 │ │ median │ 3810 │ 2826 │ 2633 │ 193 │ 961 │ 15 │ 892 │ 74 │ │ maximum │ 3810 │ 2831 │ 2638 │ 193 │ 962 │ 27 │ 892 │ 74 │ │ stdev │ 2 │ 3 │ 3 │ 0 │ 1 │ 7 │ 0 │ 0 │ └─────────┴────────────┴─────────┴───────────┴────────────┴──────────────┴───────────────────┴──────────┴────────────┘ (gcthreads, threads) = (2, 1) [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback ┌─────────┬────────────┬─────────┬───────────┬────────────┬──────────────┬───────────────────┬──────────┬────────────┐ │ │ total time │ gc time │ mark time │ sweep time │ max GC pause │ time to safepoint │ max heap │ percent gc │ │ │ ms │ ms │ ms │ ms │ ms │ us │ MB │ % │ ├─────────┼────────────┼─────────┼───────────┼────────────┼──────────────┼───────────────────┼──────────┼────────────┤ │ minimum │ 2455 │ 1476 │ 1350 │ 126 │ 488 │ 20 │ 892 │ 60 │ │ median │ 2459 │ 1478 │ 1352 │ 126 │ 489 │ 23 │ 892 │ 60 │ │ maximum │ 2465 │ 1479 │ 1352 │ 126 │ 489 │ 23 │ 893 │ 60 │ │ stdev │ 5 │ 1 │ 1 │ 0 │ 1 │ 2 │ 0 │ 0 │ └─────────┴────────────┴─────────┴───────────┴────────────┴──────────────┴───────────────────┴──────────┴────────────┘ (gcthreads, threads) = (4, 1) [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback ┌─────────┬────────────┬─────────┬───────────┬────────────┬──────────────┬───────────────────┬──────────┬────────────┐ │ │ total time │ gc time │ mark time │ sweep time │ max GC pause │ time to safepoint │ max heap │ percent gc │ │ │ ms │ ms │ ms │ ms │ ms │ us │ MB │ % │ ├─────────┼────────────┼─────────┼───────────┼────────────┼──────────────┼───────────────────┼──────────┼────────────┤ │ minimum │ 2173 │ 1194 │ 1115 │ 77 │ 463 │ 18 │ 892 │ 54 │ │ median │ 2187 │ 1194 │ 1116 │ 78 │ 463 │ 19 │ 893 │ 55 │ │ maximum │ 2217 │ 1208 │ 1130 │ 78 │ 463 │ 19 │ 893 │ 55 │ │ stdev │ 22 │ 8 │ 8 │ 1 │ 0 │ 1 │ 0 │ 0 │ └─────────┴────────────┴─────────┴───────────┴────────────┴──────────────┴───────────────────┴──────────┴────────────┘ (gcthreads, threads) = (8, 1) [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback ┌─────────┬────────────┬─────────┬───────────┬────────────┬──────────────┬───────────────────┬──────────┬────────────┐ │ │ total time │ gc time │ mark time │ sweep time │ max GC pause │ time to safepoint │ max heap │ percent gc │ │ │ ms │ ms │ ms │ ms │ ms │ us │ MB │ % │ ├─────────┼────────────┼─────────┼───────────┼────────────┼──────────────┼───────────────────┼──────────┼────────────┤ │ minimum │ 2191 │ 1212 │ 1149 │ 63 │ 539 │ 19 │ 901 │ 55 │ │ median │ 2199 │ 1219 │ 1156 │ 63 │ 540 │ 20 │ 901 │ 55 │ │ maximum │ 2201 │ 1222 │ 1157 │ 65 │ 540 │ 20 │ 901 │ 56 │ │ stdev │ 5 │ 5 │ 4 │ 1 │ 1 │ 1 │ 0 │ 0 │ └─────────┴────────────┴─────────┴───────────┴────────────┴──────────────┴───────────────────┴──────────┴────────────┘ (gcthreads, threads) = (16, 1) [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback ┌─────────┬────────────┬─────────┬───────────┬────────────┬──────────────┬───────────────────┬──────────┬────────────┐ │ │ total time │ gc time │ mark time │ sweep time │ max GC pause │ time to safepoint │ max heap │ percent gc │ │ │ ms │ ms │ ms │ ms │ ms │ us │ MB │ % │ ├─────────┼────────────┼─────────┼───────────┼────────────┼──────────────┼───────────────────┼──────────┼────────────┤ │ minimum │ 3897 │ 2916 │ 2854 │ 60 │ 1423 │ 20 │ 902 │ 75 │ │ median │ 4012 │ 3032 │ 2972 │ 62 │ 1491 │ 21 │ 904 │ 75 │ │ maximum │ 4026 │ 3106 │ 3044 │ 62 │ 1511 │ 22 │ 904 │ 77 │ │ stdev │ 70 │ 96 │ 96 │ 1 │ 46 │ 1 │ 1 │ 1 │ └─────────┴────────────┴─────────┴───────────┴────────────┴──────────────┴───────────────────┴──────────┴────────────┘ ``` This patch seems to improve performance at 16 GC threads though we still have some negative scaling for this benchmark beyond 8 GC threads (others seem fine). I didn't test whether this is indeed the optimal struct layout for the queue in this benchmark. ``` bench = "many_refs.jl" (gcthreads, threads) = (1, 1) [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback ┌─────────┬────────────┬─────────┬───────────┬────────────┬──────────────┬───────────────────┬──────────┬────────────┐ │ │ total time │ gc time │ mark time │ sweep time │ max GC pause │ time to safepoint │ max heap │ percent gc │ │ │ ms │ ms │ ms │ ms │ ms │ us │ MB │ % │ ├─────────┼────────────┼─────────┼───────────┼────────────┼──────────────┼───────────────────┼──────────┼────────────┤ │ minimum │ 3807 │ 2822 │ 2630 │ 193 │ 959 │ 15 │ 892 │ 74 │ │ median │ 3824 │ 2823 │ 2630 │ 193 │ 959 │ 15 │ 892 │ 74 │ │ maximum │ 4307 │ 3194 │ 2974 │ 220 │ 1086 │ 19 │ 892 │ 74 │ │ stdev │ 284 │ 215 │ 199 │ 16 │ 74 │ 2 │ 0 │ 0 │ └─────────┴────────────┴─────────┴───────────┴────────────┴──────────────┴───────────────────┴──────────┴────────────┘ (gcthreads, threads) = (2, 1) [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback ┌─────────┬────────────┬─────────┬───────────┬────────────┬──────────────┬───────────────────┬──────────┬────────────┐ │ │ total time │ gc time │ mark time │ sweep time │ max GC pause │ time to safepoint │ max heap │ percent gc │ │ │ ms │ ms │ ms │ ms │ ms │ us │ MB │ % │ ├─────────┼────────────┼─────────┼───────────┼────────────┼──────────────┼───────────────────┼──────────┼────────────┤ │ minimum │ 2459 │ 1476 │ 1349 │ 126 │ 488 │ 19 │ 892 │ 60 │ │ median │ 2460 │ 1479 │ 1352 │ 127 │ 488 │ 21 │ 893 │ 60 │ │ maximum │ 2770 │ 1661 │ 1526 │ 135 │ 570 │ 22 │ 893 │ 60 │ │ stdev │ 179 │ 106 │ 101 │ 5 │ 47 │ 1 │ 0 │ 0 │ └─────────┴────────────┴─────────┴───────────┴────────────┴──────────────┴───────────────────┴──────────┴────────────┘ (gcthreads, threads) = (4, 1) [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback ┌─────────┬────────────┬─────────┬───────────┬────────────┬──────────────┬───────────────────┬──────────┬────────────┐ │ │ total time │ gc time │ mark time │ sweep time │ max GC pause │ time to safepoint │ max heap │ percent gc │ │ │ ms │ ms │ ms │ ms │ ms │ us │ MB │ % │ ├─────────┼────────────┼─────────┼───────────┼────────────┼──────────────┼───────────────────┼──────────┼────────────┤ │ minimum │ 2215 │ 1233 │ 1155 │ 77 │ 485 │ 20 │ 892 │ 53 │ │ median │ 2320 │ 1248 │ 1171 │ 78 │ 492 │ 21 │ 893 │ 54 │ │ maximum │ 2372 │ 1259 │ 1176 │ 83 │ 493 │ 22 │ 893 │ 56 │ │ stdev │ 80 │ 13 │ 11 │ 3 │ 5 │ 1 │ 1 │ 1 │ └─────────┴────────────┴─────────┴───────────┴────────────┴──────────────┴───────────────────┴──────────┴────────────┘ (gcthreads, threads) = (8, 1) [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback ┌─────────┬────────────┬─────────┬───────────┬────────────┬──────────────┬───────────────────┬──────────┬────────────┐ │ │ total time │ gc time │ mark time │ sweep time │ max GC pause │ time to safepoint │ max heap │ percent gc │ │ │ ms │ ms │ ms │ ms │ ms │ us │ MB │ % │ ├─────────┼────────────┼─────────┼───────────┼────────────┼──────────────┼───────────────────┼──────────┼────────────┤ │ minimum │ 2204 │ 1220 │ 1156 │ 64 │ 544 │ 23 │ 901 │ 53 │ │ median │ 2362 │ 1249 │ 1181 │ 64 │ 553 │ 23 │ 901 │ 53 │ │ maximum │ 2374 │ 1261 │ 1196 │ 68 │ 558 │ 25 │ 901 │ 55 │ │ stdev │ 95 │ 21 │ 20 │ 2 │ 7 │ 1 │ 0 │ 1 │ └─────────┴────────────┴─────────┴───────────┴────────────┴──────────────┴───────────────────┴──────────┴────────────┘ (gcthreads, threads) = (16, 1) [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback [ Info: Setting GC memory pressure callback ┌─────────┬────────────┬─────────┬───────────┬────────────┬──────────────┬───────────────────┬──────────┬────────────┐ │ │ total time │ gc time │ mark time │ sweep time │ max GC pause │ time to safepoint │ max heap │ percent gc │ │ │ ms │ ms │ ms │ ms │ ms │ us │ MB │ % │ ├─────────┼────────────┼─────────┼───────────┼────────────┼──────────────┼───────────────────┼──────────┼────────────┤ │ minimum │ 2502 │ 1519 │ 1458 │ 62 │ 721 │ 22 │ 902 │ 58 │ │ median │ 2511 │ 1524 │ 1461 │ 63 │ 728 │ 23 │ 903 │ 61 │ │ maximum │ 2664 │ 1554 │ 1486 │ 68 │ 741 │ 25 │ 905 │ 61 │ │ stdev │ 91 │ 19 │ 16 │ 3 │ 10 │ 1 │ 2 │ 1 │ └─────────┴────────────┴─────────┴───────────┴────────────┴──────────────┴───────────────────┴──────────┴────────────┘ ``` (cherry picked from commit 973b4ff9abb6e12cada196a9c97a96b358275e16) --- src/work-stealing-queue.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/work-stealing-queue.h b/src/work-stealing-queue.h index 5902c2ac6af9f..c444ccb3da682 100644 --- a/src/work-stealing-queue.h +++ b/src/work-stealing-queue.h @@ -3,6 +3,8 @@ #ifndef WORK_STEALING_QUEUE_H #define WORK_STEALING_QUEUE_H +#include + #include "julia_atomics.h" #include "assert.h" @@ -35,10 +37,10 @@ static inline ws_array_t *create_ws_array(size_t capacity, int32_t eltsz) JL_NOT } typedef struct { - _Atomic(int64_t) top; - char _padding[JL_CACHE_BYTE_ALIGNMENT - sizeof(_Atomic(int64_t))]; - _Atomic(int64_t) bottom; // put on a separate cache line. conservatively estimate cache line size as 128 bytes - _Atomic(ws_array_t *) array; + // align to JL_CACHE_BYTE_ALIGNMENT + alignas(JL_CACHE_BYTE_ALIGNMENT) _Atomic(int64_t) top; + alignas(JL_CACHE_BYTE_ALIGNMENT) _Atomic(int64_t) bottom; + alignas(JL_CACHE_BYTE_ALIGNMENT) _Atomic(ws_array_t *) array; } ws_queue_t; static inline ws_array_t *ws_queue_push(ws_queue_t *q, void *elt, int32_t eltsz) JL_NOTSAFEPOINT From d30d8ef5d6af22b7a42eb510dcc73b1e66ea68bc Mon Sep 17 00:00:00 2001 From: Michael Goerz Date: Mon, 4 Mar 2024 11:34:27 -0500 Subject: [PATCH 14/55] Update Documenter to v1.3 for inventory writing (#53571) With the newest Documenter release, the Julia documentation will automatically have an `objects.inv` file. This file allows any other project using Documenter with the [`DocumenterInterLinks`](https://github.com/JuliaDocs/DocumenterInterLinks.jl) plugin (or any project using [Sphinx](https://www.sphinx-doc.org/en/master/)) to link directly into the Julia documentation, e.g., with ```[`Base.sort!`](@extref Julia)```. See also https://github.com/JuliaDocs/Documenter.jl/pull/2424#issuecomment-1960025462 and the following comments (cherry picked from commit 0311aa4caf461d05fb408de67dafc582e2ada9a0) --- doc/Manifest.toml | 39 +++++++++++++++++++++++++++++---------- doc/make.jl | 1 + 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/doc/Manifest.toml b/doc/Manifest.toml index b34942bbddd3a..c55c983402830 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -16,7 +16,7 @@ version = "0.4.4" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.1" +version = "1.1.2" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -26,6 +26,12 @@ version = "1.11.0" uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" version = "1.11.0" +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "59939d8a997469ee05c4b4944560a820f9ba0d73" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.4" + [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" @@ -38,10 +44,10 @@ uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" version = "0.9.3" [[deps.Documenter]] -deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "Test", "Unicode"] -git-tree-sha1 = "2613dbec8f4748273bbe30ba71fd5cb369966bac" +deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] +git-tree-sha1 = "4a40af50e8b24333b9ec6892546d9ca5724228eb" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.2.1" +version = "1.3.0" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] @@ -106,7 +112,7 @@ version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.4.0+0" +version = "8.6.0+0" [[deps.LibGit2]] deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] @@ -116,7 +122,7 @@ version = "1.11.0" [[deps.LibGit2_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.7.1+0" +version = "1.7.2+0" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] @@ -152,7 +158,7 @@ version = "0.1.2" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.2+1" +version = "2.28.6+0" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" @@ -160,7 +166,7 @@ version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2023.1.10" +version = "2023.12.12" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" @@ -184,9 +190,13 @@ uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "2.8.0" [[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" version = "1.11.0" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" [[deps.PrecompileTools]] deps = ["Preferences"] @@ -251,6 +261,15 @@ deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" version = "1.11.0" +[[deps.TranscodingStreams]] +git-tree-sha1 = "54194d92959d8ebaa8e26227dbe3cdefcdcd594f" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.10.3" +weakdeps = ["Random", "Test"] + + [deps.TranscodingStreams.extensions] + TestExt = ["Test", "Random"] + [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -268,7 +287,7 @@ version = "1.2.13+1" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.58.0+0" +version = "1.59.0+0" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] diff --git a/doc/make.jl b/doc/make.jl index 1f654649d9237..8dee832bf27b7 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -365,6 +365,7 @@ else ansicolor = true, size_threshold = 800 * 2^10, # 800 KiB size_threshold_warn = 200 * 2^10, # the manual has quite a few large pages, so we warn at 200+ KiB only + inventory_version = VERSION, ) end From bd6b7eb0ee88aa12558205165a89801ea6f9874e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 5 Mar 2024 02:07:34 +0100 Subject: [PATCH 15/55] Move parallel precompilation to Base (#53403) Parallel precompilation is more or less now required in order to use somewhat large packages unless you want to wait an obscene long time for it to complete. Right now, we even start a parallel precompilation on a package load if we notice that the package you are loading is not precompiled. This functionally has typically been implemented in Pkg but with Pkg not being in the sysimage it becomes a bit awkward because we then need to load Pkg from Base. The only real reason this functionality has been implemented in Pkg is that Pkg has some useful features for parsing environments. Moving precompilation to Base has typically been stalled on such an environment parser not existing in Base. However, in https://github.com/JuliaLang/julia/pull/46690 I started implemented code loading on top of a more up front environment parser (instead of the "incremental" one that currently exists in `loading.jl`) and we can retro fit this to be used as the basis of parallel precompilation. At some later point code loading could be implemented on top of it but that is for now considered future work. This PR thus adds the environment parser from the codeloading PR and implementes the parallel precompilation feature from Pkg on top of it (instead of on top of the `EnvCache` in Pkg). Some points to bring up here: - This copy pastes the progress bar implementation in Pkg into here. It is probably a bit excessive to use so we can simplify that significantly. - Parallel precompilation uses the `FileWatching` module to avoid different processes trying to precompile the same package concurrently. Right now, I used grab this from `Base.loaded_modules` relying on it being in the sysimage. - This removes the "suspended" functionality from the Pkg precompilation which does not try to precompile packages if they have "recently" failed which is unclear how useful it is in practice. This also requires the Serialization stdlib and uses data structures defined in Pkg so it is hard to keep when moving this to Base. --------- Co-authored-by: Ian Butterworth (cherry picked from commit 67451604cf463c912c2325eb5990ad46262468c2) --- base/Base.jl | 2 + base/loading.jl | 15 +- base/precompilation.jl | 948 +++++++++++++++++++++++++++++++++++++++++ doc/make.jl | 3 +- 4 files changed, 960 insertions(+), 8 deletions(-) create mode 100644 base/precompilation.jl diff --git a/base/Base.jl b/base/Base.jl index e02b132aae6e4..c6a1113996fc3 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -542,6 +542,8 @@ if isdefined(Core, :Compiler) && is_primary_base_module Docs.loaddocs(Core.Compiler.CoreDocs.DOCS) end +include("precompilation.jl") + # finally, now make `include` point to the full version for m in methods(include) delete_method(m) diff --git a/base/loading.jl b/base/loading.jl index 2be97237238dd..5f3c846131810 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2147,8 +2147,6 @@ end require(uuidkey::PkgId) = @lock require_lock _require_prelocked(uuidkey) -const REPL_PKGID = PkgId(UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL") - function _require_prelocked(uuidkey::PkgId, env=nothing) if _require_world_age[] != typemax(UInt) Base.invoke_in_world(_require_world_age[], __require_prelocked, uuidkey, env) @@ -2262,8 +2260,9 @@ function set_pkgorigin_version_path(pkg::PkgId, path::String) nothing end -# A hook to allow code load to use Pkg.precompile +# Unused const PKG_PRECOMPILE_HOOK = Ref{Function}() +disable_parallel_precompile::Bool = false # Returns `nothing` or the new(ish) module function _require(pkg::PkgId, env=nothing) @@ -2284,7 +2283,7 @@ function _require(pkg::PkgId, env=nothing) end set_pkgorigin_version_path(pkg, path) - pkg_precompile_attempted = false # being safe to avoid getting stuck in a Pkg.precompile loop + parallel_precompile_attempted = false # being safe to avoid getting stuck in a precompilepkgs loop reasons = Dict{String,Int}() # attempt to load the module file via the precompile cache locations if JLOptions().use_compiled_modules != 0 @@ -2314,11 +2313,13 @@ function _require(pkg::PkgId, env=nothing) if JLOptions().use_compiled_modules == 1 if !generating_output(#=incremental=#false) - if !pkg_precompile_attempted && isinteractive() && isassigned(PKG_PRECOMPILE_HOOK) - pkg_precompile_attempted = true + project = active_project() + if !generating_output() && !parallel_precompile_attempted && !disable_parallel_precompile && @isdefined(Precompilation) && project !== nothing && + isfile(project) && project_file_manifest_path(project) !== nothing + parallel_precompile_attempted = true unlock(require_lock) try - @invokelatest PKG_PRECOMPILE_HOOK[](pkg.name, _from_loading = true) + Precompilation.precompilepkgs([pkg.name]; _from_loading=true) finally lock(require_lock) end diff --git a/base/precompilation.jl b/base/precompilation.jl new file mode 100644 index 0000000000000..4efd35836a29d --- /dev/null +++ b/base/precompilation.jl @@ -0,0 +1,948 @@ +module Precompilation + +using Base: PkgId, UUID, SHA1, parsed_toml, project_file_name_uuid, project_names, + project_file_manifest_path, get_deps, preferences_names, isaccessibledir, isfile_casesensitive + +# This is currently only used for pkgprecompile but the plan is to use this in code loading in the future +# see the `kc/codeloading2.0` branch +struct ExplicitEnv + path::String + project_deps::Dict{String, UUID} # [deps] in Project.toml + project_weakdeps::Dict{String, UUID} # [weakdeps] in Project.toml + project_extras::Dict{String, UUID} # [extras] in Project.toml + project_extensions::Dict{String, Vector{UUID}} # [exts] in Project.toml + deps::Dict{UUID, Vector{UUID}} # all dependencies in Manifest.toml + weakdeps::Dict{UUID, Vector{UUID}} # all weak dependencies in Manifest.toml + extensions::Dict{UUID, Dict{String, Vector{UUID}}} + # Lookup name for a UUID + names::Dict{UUID, String} + lookup_strategy::Dict{UUID, Union{ + SHA1, # `git-tree-sha1` entry + String, # `path` entry + Nothing, # stdlib (no `path` nor `git-tree-sha1`) + Missing}} # not present in the manifest + #prefs::Union{Nothing, Dict{String, Any}} + #local_prefs::Union{Nothing, Dict{String, Any}} +end + +function ExplicitEnv(envpath::String=Base.active_project()) + envpath = abspath(envpath) + project_d = parsed_toml(envpath) + + # TODO: Perhaps verify that two packages with the same UUID do not have different names? + names = Dict{UUID, String}() + project_uuid_to_name = Dict{String, UUID}() + + project_deps = Dict{String, UUID}() + project_weakdeps = Dict{String, UUID}() + project_extras = Dict{String, UUID}() + + # Collect all direct dependencies of the project + for key in ["deps", "weakdeps", "extras"] + for (name, _uuid) in get(Dict{String, Any}, project_d, key)::Dict{String, Any} + v = key == "deps" ? project_deps : + key == "weakdeps" ? project_weakdeps : + key == "extras" ? project_extras : + error() + uuid = UUID(_uuid) + v[name] = uuid + names[UUID(uuid)] = name + project_uuid_to_name[name] = UUID(uuid) + end + end + + project_extensions = Dict{String, Vector{UUID}}() + # Collect all extensions of the project + for (name, triggers::Union{String, Vector{String}}) in get(Dict{String, Any}, project_d, "extensions")::Dict{String, Any} + if triggers isa String + triggers = [triggers] + end + uuids = UUID[] + for trigger in triggers + uuid = get(project_uuid_to_name, trigger, nothing) + if uuid === nothing + error("Trigger $trigger for extension $name not found in project") + end + push!(uuids, uuid) + end + project_extensions[name] = uuids + end + + # This project might be a package, in that case, that is also a "dependency" + # of the project. + proj_name = get(project_d, "name", nothing)::Union{String, Nothing} + _proj_uuid = get(project_d, "uuid", nothing)::Union{String, Nothing} + proj_uuid = _proj_uuid === nothing ? nothing : UUID(_proj_uuid) + + if proj_name !== nothing && proj_uuid !== nothing + # TODO: Error on missing uuid? + project_deps[proj_name] = UUID(proj_uuid) + names[UUID(proj_uuid)] = proj_name + end + + manifest = project_file_manifest_path(envpath) + manifest_d = manifest === nothing ? Dict{String, Any}() : parsed_toml(manifest) + + # Dependencies in a manifest can either be stored compressed (when name is unique among all packages) + # in which case it is a `Vector{String}` or expanded where it is a `name => uuid` mapping. + deps = Dict{UUID, Union{Vector{String}, Vector{UUID}}}() + weakdeps = Dict{UUID, Union{Vector{String}, Vector{UUID}}}() + extensions = Dict{UUID, Dict{String, Vector{String}}}() + name_to_uuid = Dict{String, UUID}() + lookup_strategy = Dict{UUID, Union{SHA1, String, Nothing, Missing}}() + + sizehint!(deps, length(manifest_d)) + sizehint!(weakdeps, length(manifest_d)) + sizehint!(extensions, length(manifest_d)) + sizehint!(name_to_uuid, length(manifest_d)) + sizehint!(lookup_strategy, length(manifest_d)) + + for (name, pkg_infos) in get_deps(manifest_d) + pkg_infos = pkg_infos::Vector{Any} + for pkg_info in pkg_infos + m_uuid = UUID(pkg_info["uuid"]::String) + + # If we have multiple packages with the same name we will overwrite things here + # but that is fine since we will only use the information in here for packages + # with unique names + names[m_uuid] = name + name_to_uuid[name] = m_uuid + + for key in ["deps", "weakdeps"] + deps_pkg = get(Vector{String}, pkg_info, key)::Union{Vector{String}, Dict{String, Any}} + d = key == "deps" ? deps : + key == "weakdeps" ? weakdeps : + error() + + # Compressed format with unique names: + if deps_pkg isa Vector{String} + d[m_uuid] = deps_pkg + # Expanded format: + else + uuids = UUID[] + for (name_dep, _dep_uuid::String) in deps_pkg + dep_uuid = UUID(_dep_uuid) + push!(uuids, dep_uuid) + names[dep_uuid] = name_dep + end + d[m_uuid] = uuids + end + end + + # Extensions + deps_pkg = get(Dict{String, Any}, pkg_info, "extensions")::Dict{String, Any} + for (ext, triggers) in deps_pkg + triggers = triggers::Union{String, Vector{String}} + if triggers isa String + triggers = [triggers] + end + deps_pkg[ext] = triggers + end + extensions[m_uuid] = deps_pkg + + # Determine strategy to find package + lookup_strat = begin + if (path = get(pkg_info, "path", nothing)::Union{String, Nothing}) !== nothing + path + elseif (git_tree_sha_str = get(pkg_info, "git-tree-sha1", nothing)::Union{String, Nothing}) !== nothing + SHA1(git_tree_sha_str) + else + nothing + end + end + lookup_strategy[m_uuid] = lookup_strat + end + end + + # No matter if the deps were stored compressed or not in the manifest, + # we internally store them expanded + deps_expanded = Dict{UUID, Vector{UUID}}() + weakdeps_expanded = Dict{UUID, Vector{UUID}}() + extensions_expanded = Dict{UUID, Dict{String, Vector{UUID}}}() + sizehint!(deps_expanded, length(deps)) + sizehint!(weakdeps_expanded, length(deps)) + sizehint!(extensions_expanded, length(deps)) + + if proj_name !== nothing + deps_expanded[proj_uuid] = filter!(!=(proj_uuid), collect(values(project_deps))) + extensions_expanded[proj_uuid] = project_extensions + path = get(project_d, "path", nothing) + entry_point = path !== nothing ? path : dirname(envpath) + lookup_strategy[proj_uuid] = entry_point + end + + for key in ["deps", "weakdeps"] + d = key == "deps" ? deps : + key == "weakdeps" ? weakdeps : + error() + d_expanded = key == "deps" ? deps_expanded : + key == "weakdeps" ? weakdeps_expanded : + error() + for (pkg, deps) in d + # dependencies was already expanded so use it directly: + if deps isa Vector{UUID} + d_expanded[pkg] = deps + for dep in deps + name_to_uuid[names[dep]] = dep + end + # find the (unique) UUID associated with the name + else + deps_pkg = UUID[] + sizehint!(deps_pkg, length(deps)) + for dep in deps + push!(deps_pkg, name_to_uuid[dep]) + end + d_expanded[pkg] = deps_pkg + end + end + end + + for (pkg, exts) in extensions + exts_expanded = Dict{String, Vector{UUID}}() + for (ext, triggers) in exts + triggers_expanded = UUID[] + sizehint!(triggers_expanded, length(triggers)) + for trigger in triggers + push!(triggers_expanded, name_to_uuid[trigger]) + end + exts_expanded[ext] = triggers_expanded + end + extensions_expanded[pkg] = exts_expanded + end + + # Everything that does not yet have a lookup_strategy is missing from the manifest + for (_, uuid) in project_deps + get!(lookup_strategy, uuid, missing) + end + + #= + # Preferences: + prefs = get(project_d, "preferences", nothing) + + # `(Julia)LocalPreferences.toml` + project_dir = dirname(envpath) + local_prefs = nothing + for name in preferences_names + toml_path = joinpath(project_dir, name) + if isfile(toml_path) + local_prefs = parsed_toml(toml_path) + break + end + end + =# + + return ExplicitEnv(envpath, project_deps, project_weakdeps, project_extras, + project_extensions, deps_expanded, weakdeps_expanded, extensions_expanded, + names, lookup_strategy, #=prefs, local_prefs=#) +end + +## PROGRESS BAR + +# using Printf +Base.@kwdef mutable struct MiniProgressBar + max::Int = 1.0 + header::String = "" + color::Symbol = :nothing + width::Int = 40 + current::Int = 0.0 + prev::Int = 0.0 + has_shown::Bool = false + time_shown::Float64 = 0.0 + percentage::Bool = true + always_reprint::Bool = false + indent::Int = 4 +end + +const PROGRESS_BAR_TIME_GRANULARITY = Ref(1 / 30.0) # 30 fps +const PROGRESS_BAR_PERCENTAGE_GRANULARITY = Ref(0.1) + +function start_progress(io::IO, _::MiniProgressBar) + ansi_disablecursor = "\e[?25l" + print(io, ansi_disablecursor) +end + +function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagereturn=true) + if p.max == 0 + perc = 0.0 + prev_perc = 0.0 + else + perc = p.current / p.max * 100 + prev_perc = p.prev / p.max * 100 + end + # Bail early if we are not updating the progress bar, + # Saves printing to the terminal + if !p.always_reprint && p.has_shown && !((perc - prev_perc) > PROGRESS_BAR_PERCENTAGE_GRANULARITY[]) + return + end + t = time() + if p.has_shown && (t - p.time_shown) < PROGRESS_BAR_TIME_GRANULARITY[] + return + end + p.time_shown = t + p.prev = p.current + p.has_shown = true + + progress_text = if false # p.percentage + # @sprintf "%2.1f %%" perc + else + string(p.current, "/", p.max) + end + termwidth = @something termwidth displaysize(io)[2] + max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10 , p.width)) + n_filled = ceil(Int, max_progress_width * perc / 100) + n_left = max_progress_width - n_filled + to_print = sprint(; context=io) do io + print(io, " "^p.indent) + printstyled(io, p.header, color=p.color, bold=true) + print(io, " [") + print(io, "="^n_filled, ">") + print(io, " "^n_left, "] ", ) + print(io, progress_text) + carriagereturn && print(io, "\r") + end + # Print everything in one call + print(io, to_print) +end + +function end_progress(io, p::MiniProgressBar) + ansi_enablecursor = "\e[?25h" + ansi_clearline = "\e[2K" + print(io, ansi_enablecursor * ansi_clearline) +end + +function print_progress_bottom(io::IO) + ansi_clearline = "\e[2K" + ansi_movecol1 = "\e[1G" + ansi_moveup(n::Int) = string("\e[", n, "A") + print(io, "\e[S" * ansi_moveup(1) * ansi_clearline * ansi_movecol1) +end + + +############ +struct PkgPrecompileError <: Exception + msg::String +end +Base.showerror(io::IO, err::PkgPrecompileError) = print(io, err.msg) +# This needs a show method to make `julia> err` show nicely +Base.show(io::IO, err::PkgPrecompileError) = print(io, "PkgPrecompileError: ", err.msg) + +import Base: StaleCacheKey + +can_fancyprint(io::IO) = io isa Base.TTY && (get(ENV, "CI", nothing) != "true") + +function printpkgstyle(io, header, msg; color=:light_green) + printstyled(io, header; color, bold=true) + println(io, " ", msg) +end + + +function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false, + strict::Bool=false, warn_loaded = true, timing::Bool = false, + _from_loading::Bool=false, flags_cacheflags::Pair{Cmd, Base.CacheFlags}=(``=>Base.CacheFlags()), io::IO=stderr) + time_start = time_ns() + + env = ExplicitEnv() + + flags, cacheflags = flags_cacheflags + + # Windows sometimes hits a ReadOnlyMemoryError, so we halve the default number of tasks. Issue #2323 + # TODO: Investigate why this happens in windows and restore the full task limit + default_num_tasks = Sys.iswindows() ? div(Sys.CPU_THREADS::Int, 2) + 1 : Sys.CPU_THREADS::Int + 1 + default_num_tasks = min(default_num_tasks, 16) # limit for better stability on shared resource systems + + num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks))) + parallel_limiter = Base.Semaphore(num_tasks) + + # asking for timing disables fancy mode, as timing is shown in non-fancy mode + fancyprint = can_fancyprint(io) && !timing + + if _from_loading && !Sys.isinteractive() + io = devnull + end + + hascolor = get(io, :color, false)::Bool + color_string(cstr::String, col::Union{Int64, Symbol}) = _color_string(cstr, col, hascolor) + + direct_deps = [ + Base.PkgId(uuid, name) + for (name, uuid) in env.project_deps if !Base.in_sysimage(Base.PkgId(uuid, name)) + ] + stale_cache = Dict{StaleCacheKey, Bool}() + exts = Dict{Base.PkgId, String}() # ext -> parent + # make a flat map of each dep and its direct deps + depsmap = Dict{Base.PkgId, Vector{Base.PkgId}}() + pkg_exts_map = Dict{Base.PkgId, Vector{Base.PkgId}}() + + for (dep, deps) in env.deps + pkg = Base.PkgId(dep, env.names[dep]) + Base.in_sysimage(pkg) && continue + deps = [Base.PkgId(x, env.names[x]) for x in deps] + depsmap[pkg] = filter!(!Base.in_sysimage, deps) + # add any extensions + pkg_exts = Dict{Base.PkgId, Vector{Base.PkgId}}() + prev_ext = nothing + for (ext_name, extdep_uuids) in env.extensions[dep] + ext_deps = Base.PkgId[] + push!(ext_deps, pkg) # depends on parent package + all_extdeps_available = true + for extdep_uuid in extdep_uuids + extdep_name = env.names[extdep_uuid] + if extdep_uuid in keys(env.deps) || Base.in_sysimage(Base.PkgId(extdep_uuid, extdep_name)) + push!(ext_deps, Base.PkgId(extdep_uuid, extdep_name)) + else + all_extdeps_available = false + break + end + end + all_extdeps_available || continue + if prev_ext isa Base.PkgId + # also make the exts depend on eachother sequentially to avoid race + push!(ext_deps, prev_ext) + end + ext_uuid = Base.uuid5(pkg.uuid, ext_name) + ext = Base.PkgId(ext_uuid, ext_name) + prev_ext = ext + filter!(!Base.in_sysimage, ext_deps) + depsmap[ext] = ext_deps + exts[ext] = pkg.name + pkg_exts[ext] = ext_deps + end + if !isempty(pkg_exts) + pkg_exts_map[pkg] = collect(keys(pkg_exts)) + end + end + + @debug "precompile: deps collected" _group=:precompile + # this loop must be run after the full depsmap has been populated + for (pkg, pkg_exts) in pkg_exts_map + # find any packages that depend on the extension(s)'s deps and replace those deps in their deps list with the extension(s), + # basically injecting the extension into the precompile order in the graph, to avoid race to precompile extensions + for (_pkg, deps) in depsmap # for each manifest dep + if !in(_pkg, keys(exts)) && pkg in deps # if not an extension and depends on pkg + append!(deps, pkg_exts) # add the package extensions to deps + filter!(!isequal(pkg), deps) # remove the pkg from deps + end + end + end + @debug "precompile: extensions collected" _group=:precompile + + # return early if no deps + if isempty(depsmap) + if isempty(pkgs) + return + elseif _from_loading + # if called from loading precompilation it may be a package from another environment stack so + # don't error and allow serial precompilation to try + # TODO: actually handle packages from other envs in the stack + return + else + error("No direct dependencies outside of the sysimage found matching $(repr([p.name for p in pkgs]))") + end + end + + # initialize signalling + started = Dict{Base.PkgId,Bool}() + was_processed = Dict{Base.PkgId,Base.Event}() + was_recompiled = Dict{Base.PkgId,Bool}() + for pkgid in keys(depsmap) + started[pkgid] = false + was_processed[pkgid] = Base.Event() + was_recompiled[pkgid] = false + end + @debug "precompile: signalling initialized" _group=:precompile + + + # find and guard against circular deps + circular_deps = Base.PkgId[] + # Three states + # !haskey -> never visited + # true -> cannot be compiled due to a cycle (or not yet determined) + # false -> not depending on a cycle + could_be_cycle = Dict{Base.PkgId, Bool}() + function scan_pkg!(pkg, dmap) + did_visit_dep = true + inpath = get!(could_be_cycle, pkg) do + did_visit_dep = false + return true + end + if did_visit_dep ? inpath : scan_deps!(pkg, dmap) + # Found a cycle. Delete this and all parents + return true + end + return false + end + function scan_deps!(pkg, dmap) + for dep in dmap[pkg] + scan_pkg!(dep, dmap) && return true + end + could_be_cycle[pkg] = false + return false + end + for pkg in keys(depsmap) + if scan_pkg!(pkg, depsmap) + push!(circular_deps, pkg) + notify(was_processed[pkg]) + end + end + if !isempty(circular_deps) + @warn """Circular dependency detected. Precompilation will be skipped for:\n $(join(string.(circular_deps), "\n "))""" + end + @debug "precompile: circular dep check done" _group=:precompile + + # if a list of packages is given, restrict to dependencies of given packages + if !isempty(pkgs) + function collect_all_deps(depsmap, dep, alldeps=Set{Base.PkgId}()) + for _dep in depsmap[dep] + if !(_dep in alldeps) + push!(alldeps, _dep) + collect_all_deps(depsmap, _dep, alldeps) + end + end + return alldeps + end + keep = Set{Base.PkgId}() + for dep in depsmap + dep_pkgid = first(dep) + if dep_pkgid.name in pkgs + push!(keep, dep_pkgid) + collect_all_deps(depsmap, dep_pkgid, keep) + end + end + for ext in keys(exts) + if issubset(collect_all_deps(depsmap, ext), keep) # if all extension deps are kept + push!(keep, ext) + end + end + filter!(d->in(first(d), keep), depsmap) + if isempty(depsmap) + if _from_loading + # if called from loading precompilation it may be a package from another environment stack so + # don't error and allow serial precompilation to try + # TODO: actually handle packages from other envs in the stack + return + else + error("No direct dependencies outside of the sysimage found matching $(repr(pkgs))") + end + end + target = join(pkgs, ", ") + else + target = "project..." + end + @debug "precompile: packages filtered" _group=:precompile + + pkg_queue = Base.PkgId[] + failed_deps = Dict{Base.PkgId, String}() + skipped_deps = Base.PkgId[] + precomperr_deps = Base.PkgId[] # packages that may succeed after a restart (i.e. loaded packages with no cache file) + + print_lock = io isa Base.LibuvStream ? io.lock::ReentrantLock : ReentrantLock() + first_started = Base.Event() + printloop_should_exit::Bool = !fancyprint # exit print loop immediately if not fancy printing + interrupted_or_done = Base.Event() + + ansi_moveup(n::Int) = string("\e[", n, "A") + ansi_movecol1 = "\e[1G" + ansi_cleartoend = "\e[0J" + ansi_cleartoendofline = "\e[0K" + ansi_enablecursor = "\e[?25h" + ansi_disablecursor = "\e[?25l" + n_done::Int = 0 + n_already_precomp::Int = 0 + n_loaded::Int = 0 + interrupted = false + + function handle_interrupt(err, in_printloop = false) + notify(interrupted_or_done) + in_printloop || wait(t_print) # wait to let the print loop cease first + if err isa InterruptException + lock(print_lock) do + println(io, " Interrupted: Exiting precompilation...") + end + interrupted = true + return true + else + return false + end + end + + std_outputs = Dict{Base.PkgId,String}() + taskwaiting = Set{Base.PkgId}() + pkgspidlocked = Dict{Base.PkgId,String}() + pkg_liveprinted = nothing + + function monitor_std(pkg, pipe; single_requested_pkg=false) + try + liveprinting = false + while !eof(pipe) + str = readline(pipe, keep=true) + if single_requested_pkg && (liveprinting || !isempty(str)) + lock(print_lock) do + if !liveprinting + printpkgstyle(io, :Info, "Given $(pkg.name) was explicitly requested, output will be shown live $ansi_cleartoendofline", + color = Base.info_color()) + liveprinting = true + pkg_liveprinted = pkg + end + print(io, ansi_cleartoendofline, str) + end + end + std_outputs[pkg] = string(get(std_outputs, pkg, ""), str) + if !in(pkg, taskwaiting) && occursin("waiting for IO to finish", str) + !fancyprint && lock(print_lock) do + println(io, pkg.name, color_string(" Waiting for background task / IO / timer.", Base.warn_color())) + end + push!(taskwaiting, pkg) + end + if !fancyprint && in(pkg, taskwaiting) + lock(print_lock) do + print(io, str) + end + end + end + catch err + err isa InterruptException || rethrow() + end + end + + ## fancy print loop + t_print = @async begin + try + wait(first_started) + (isempty(pkg_queue) || interrupted_or_done.set) && return + fancyprint && lock(print_lock) do + printpkgstyle(io, :Precompiling, target) + print(io, ansi_disablecursor) + end + t = Timer(0; interval=1/10) + anim_chars = ["◐","◓","◑","◒"] + i = 1 + last_length = 0 + bar = MiniProgressBar(; indent=2, header = "Progress", color = Base.info_color(), percentage=false, always_reprint=true) + n_total = length(depsmap) + bar.max = n_total - n_already_precomp + final_loop = false + n_print_rows = 0 + while !printloop_should_exit + lock(print_lock) do + term_size = Base.displaysize(io)::Tuple{Int,Int} + num_deps_show = term_size[1] - 3 + pkg_queue_show = if !interrupted_or_done.set && length(pkg_queue) > num_deps_show + last(pkg_queue, num_deps_show) + else + pkg_queue + end + str_ = sprint() do iostr + if i > 1 + print(iostr, ansi_cleartoend) + end + bar.current = n_done - n_already_precomp + bar.max = n_total - n_already_precomp + # when sizing to the terminal width subtract a little to give some tolerance to resizing the + # window between print cycles + termwidth = displaysize(io)[2] - 4 + if !final_loop + str = sprint(io -> show_progress(io, bar; termwidth, carriagereturn=false); context=io) + print(iostr, Base._truncate_at_width_or_chars(true, str, termwidth), "\n") + end + for dep in pkg_queue_show + loaded = warn_loaded && haskey(Base.loaded_modules, dep) + _name = haskey(exts, dep) ? string(exts[dep], " → ", dep.name) : dep.name + name = dep in direct_deps ? _name : string(color_string(_name, :light_black)) + line = if dep in precomperr_deps + string(color_string(" ? ", Base.warn_color()), name) + elseif haskey(failed_deps, dep) + string(color_string(" ✗ ", Base.error_color()), name) + elseif was_recompiled[dep] + !loaded && interrupted_or_done.set && continue + loaded || @async begin # keep successful deps visible for short period + sleep(1); + filter!(!isequal(dep), pkg_queue) + end + string(color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) + elseif started[dep] + # Offset each spinner animation using the first character in the package name as the seed. + # If not offset, on larger terminal fonts it looks odd that they all sync-up + anim_char = anim_chars[(i + Int(dep.name[1])) % length(anim_chars) + 1] + anim_char_colored = dep in direct_deps ? anim_char : color_string(anim_char, :light_black) + waiting = if haskey(pkgspidlocked, dep) + who_has_lock = pkgspidlocked[dep] + color_string(" Being precompiled by $(who_has_lock)", Base.info_color()) + elseif dep in taskwaiting + color_string(" Waiting for background task / IO / timer. Interrupt to inspect", Base.warn_color()) + else + "" + end + string(" ", anim_char_colored, " ", name, waiting) + else + string(" ", name) + end + println(iostr, Base._truncate_at_width_or_chars(true, line, termwidth)) + end + end + last_length = length(pkg_queue_show) + n_print_rows = count("\n", str_) + print(io, str_) + printloop_should_exit = interrupted_or_done.set && final_loop + final_loop = interrupted_or_done.set # ensures one more loop to tidy last task after finish + i += 1 + printloop_should_exit || print(io, ansi_moveup(n_print_rows), ansi_movecol1) + end + wait(t) + end + catch err + handle_interrupt(err, true) || rethrow() + finally + fancyprint && print(io, ansi_enablecursor) + end + end + tasks = Task[] + if !_from_loading + Base.LOADING_CACHE[] = Base.LoadingCache() + end + @debug "precompile: starting precompilation loop" _group=:precompile + ## precompilation loop + for (pkg, deps) in depsmap + cachepaths = Base.find_all_in_cache_path(pkg) + sourcepath = Base.locate_package(pkg) + if sourcepath === nothing + failed_deps[pkg] = "Error: Missing source file for $(pkg)" + notify(was_processed[pkg]) + continue + end + # Heuristic for when precompilation is disabled + if occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcepath, String)) + notify(was_processed[pkg]) + continue + end + + single_requested_pkg = length(pkgs) == 1 && only(pkgs) == pkg.name + + task = @async begin + try + loaded = haskey(Base.loaded_modules, pkg) + for dep in deps # wait for deps to finish + wait(was_processed[dep]) + end + + circular = pkg in circular_deps + is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepaths, sourcepath, flags=cacheflags) + if !circular && is_stale + Base.acquire(parallel_limiter) + is_direct_dep = pkg in direct_deps + + # std monitoring + std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) + t_monitor = @async monitor_std(pkg, std_pipe; single_requested_pkg) + + _name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name + name = is_direct_dep ? _name : string(color_string(_name, :light_black)) + !fancyprint && lock(print_lock) do + isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) + end + push!(pkg_queue, pkg) + started[pkg] = true + fancyprint && notify(first_started) + if interrupted_or_done.set + notify(was_processed[pkg]) + Base.release(parallel_limiter) + return + end + try + # allows processes to wait if another process is precompiling a given package to + # a functionally identical package cache (except for preferences, which may differ) + t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg, pkgspidlocked, hascolor, cacheflags) do + Base.with_logger(Base.NullLogger()) do + # The false here means we ignore loaded modules, so precompile for a fresh session + Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags) + end + end + if ret isa Base.PrecompilableError + push!(precomperr_deps, pkg) + !fancyprint && lock(print_lock) do + println(io, _timing_string(t), color_string(" ? ", Base.warn_color()), name) + end + else + !fancyprint && lock(print_lock) do + println(io, _timing_string(t), color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) + end + was_recompiled[pkg] = true + end + loaded && (n_loaded += 1) + catch err + close(std_pipe.in) # close pipe to end the std output monitor + wait(t_monitor) + if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file")) + failed_deps[pkg] = (strict || is_direct_dep) ? string(sprint(showerror, err), "\n", strip(get(std_outputs, pkg, ""))) : "" + delete!(std_outputs, pkg) # so it's not shown as warnings, given error report + !fancyprint && lock(print_lock) do + println(io, " "^9, color_string(" ✗ ", Base.error_color()), name) + end + else + rethrow() + end + finally + isopen(std_pipe.in) && close(std_pipe.in) # close pipe to end the std output monitor + wait(t_monitor) + Base.release(parallel_limiter) + end + else + is_stale || (n_already_precomp += 1) + end + n_done += 1 + notify(was_processed[pkg]) + catch err_outer + # For debugging: + # @error "Task failed" exception=(err_outer, catch_backtrace()) + handle_interrupt(err_outer) || rethrow() + notify(was_processed[pkg]) + finally + filter!(!istaskdone, tasks) + length(tasks) == 1 && notify(interrupted_or_done) + end + end + push!(tasks, task) + end + isempty(tasks) && notify(interrupted_or_done) + try + wait(interrupted_or_done) + catch err + handle_interrupt(err) || rethrow() + finally + Base.LOADING_CACHE[] = nothing + end + notify(first_started) # in cases of no-op or !fancyprint + fancyprint && wait(t_print) + quick_exit = !all(istaskdone, tasks) || interrupted # if some not finished internal error is likely + seconds_elapsed = round(Int, (time_ns() - time_start) / 1e9) + ndeps = count(values(was_recompiled)) + if ndeps > 0 || !isempty(failed_deps) || (quick_exit && !isempty(std_outputs)) + str = sprint() do iostr + if !quick_exit + plural = ndeps == 1 ? "y" : "ies" + print(iostr, " $(ndeps) dependenc$(plural) successfully precompiled in $(seconds_elapsed) seconds") + if n_already_precomp > 0 || !isempty(circular_deps) || !isempty(skipped_deps) + n_already_precomp > 0 && (print(iostr, ". $n_already_precomp already precompiled")) + !isempty(circular_deps) && (print(iostr, ". $(length(circular_deps)) skipped due to circular dependency")) + !isempty(skipped_deps) && (print(iostr, ". $(length(skipped_deps)) skipped during auto due to previous errors")) + print(iostr, ".") + end + if n_loaded > 0 + plural1 = n_loaded == 1 ? "y" : "ies" + plural2 = n_loaded == 1 ? "a different version is" : "different versions are" + plural3 = n_loaded == 1 ? "" : "s" + print(iostr, "\n ", + color_string(string(n_loaded), Base.warn_color()), + " dependenc$(plural1) precompiled but $(plural2) currently loaded. Restart julia to access the new version$(plural3)" + ) + end + if !isempty(precomperr_deps) + pluralpc = length(precomperr_deps) == 1 ? "y" : "ies" + print(iostr, "\n ", + color_string(string(length(precomperr_deps)), Base.warn_color()), + " dependenc$(pluralpc) failed but may be precompilable after restarting julia" + ) + end + end + # show any stderr output, even if Pkg.precompile has been interrupted (quick_exit=true), given user may be + # interrupting a hanging precompile job with stderr output. julia#48371 + filter!(kv -> !isempty(strip(last(kv))), std_outputs) # remove empty output + if !isempty(std_outputs) + plural1 = length(std_outputs) == 1 ? "y" : "ies" + plural2 = length(std_outputs) == 1 ? "" : "s" + print(iostr, "\n ", color_string("$(length(std_outputs))", Base.warn_color()), " dependenc$(plural1) had output during precompilation:") + for (pkgid, err) in std_outputs + err = if pkgid == pkg_liveprinted + "[Output was shown above]" + else + join(split(strip(err), "\n"), color_string("\n│ ", Base.warn_color())) + end + name = haskey(exts, pkgid) ? string(exts[pkgid], " → ", pkgid.name) : pkgid.name + print(iostr, color_string("\n┌ ", Base.warn_color()), name, color_string("\n│ ", Base.warn_color()), err, color_string("\n└ ", Base.warn_color())) + end + end + end + let str=str + lock(print_lock) do + println(io, str) + end + end + quick_exit && return + err_str = "" + n_direct_errs = 0 + for (dep, err) in failed_deps + if strict || (dep in direct_deps) + err_str = string(err_str, "\n$dep\n\n$err", (n_direct_errs > 0 ? "\n" : "")) + n_direct_errs += 1 + end + end + if err_str != "" + pluralde = n_direct_errs == 1 ? "y" : "ies" + direct = strict ? "" : "direct " + err_msg = "The following $n_direct_errs $(direct)dependenc$(pluralde) failed to precompile:\n$(err_str[1:end-1])" + if internal_call # aka. auto-precompilation + if isinteractive() && !get(ENV, "CI", false) + plural1 = length(failed_deps) == 1 ? "y" : "ies" + println(io, " ", color_string("$(length(failed_deps))", Base.error_color()), " dependenc$(plural1) errored.") + println(io, " For a report of the errors see `julia> err`. To retry use `pkg> precompile`") + setglobal!(Base.MainInclude, :err, PkgPrecompileError(err_msg)) + else + # auto-precompilation shouldn't throw but if the user can't easily access the + # error messages, just show them + print(io, "\n", err_msg) + end + else + println(io) + error(err_msg) + end + end + end + nothing +end + +_timing_string(t) = string(lpad(round(t * 1e3, digits = 1), 9), " ms") + +function _color_string(cstr::String, col::Union{Int64, Symbol}, hascolor) + if hascolor + enable_ansi = get(Base.text_colors, col, Base.text_colors[:default]) + disable_ansi = get(Base.disable_text_style, col, Base.text_colors[:default]) + return string(enable_ansi, cstr, disable_ansi) + else + return cstr + end +end + +# Can be merged with `maybe_cachefile_lock` in loading? +function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg::Base.PkgId, pkgspidlocked::Dict{Base.PkgId,String}, hascolor, cacheflags) + FileWatching = Base.loaded_modules[Base.PkgId(Base.UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"), "FileWatching")] + stale_age = Base.compilecache_pidlock_stale_age + pidfile = Base.compilecache_pidfile_path(pkg) + cachefile = FileWatching.trymkpidlock(f, pidfile; stale_age) + if cachefile === false + pid, hostname, age = FileWatching.Pidfile.parse_pidfile(pidfile) + pkgspidlocked[pkg] = if isempty(hostname) || hostname == gethostname() + if pid == getpid() + "an async task in this process (pidfile: $pidfile)" + else + "another process (pid: $pid, pidfile: $pidfile)" + end + else + "another machine (hostname: $hostname, pid: $pid, pidfile: $pidfile)" + end + !fancyprint && lock(print_lock) do + println(io, " ", pkg.name, _color_string(" Being precompiled by $(pkgspidlocked[pkg])", Base.info_color(), hascolor)) + end + # wait until the lock is available + FileWatching.mkpidlock(pidfile; stale_age) do + # double-check in case the other process crashed or the lock expired + if Base.isprecompiled(pkg; ignore_loaded=true, flags=cacheflags) # don't use caches for this as the env state will have changed + return nothing # returning nothing indicates a process waited for another + else + delete!(pkgspidlocked, pkg) + return f() # precompile + end + end + end + return cachefile +end + +end diff --git a/doc/make.jl b/doc/make.jl index 8dee832bf27b7..e06fa1deb8f42 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -3,7 +3,8 @@ Base.ACTIVE_PROJECT[] = nothing empty!(LOAD_PATH) push!(LOAD_PATH, @__DIR__, "@stdlib") empty!(DEPOT_PATH) -pushfirst!(DEPOT_PATH, joinpath(@__DIR__, "deps")) +push!(DEPOT_PATH, joinpath(@__DIR__, "deps")) +push!(DEPOT_PATH, abspath(Sys.BINDIR, "..", "share", "julia")) using Pkg Pkg.instantiate() From f8fd7fda238580f68d52f009e47b2cb18d107004 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 5 Mar 2024 10:59:32 +0100 Subject: [PATCH 16/55] add back `unsafe_convert` to pointer for arrays (#53589) This caused a pretty big breakage in the ecosystem (~50 packages) and is according to Jameson "Too breaking for no necessary reason". So let's add it back. --------- Co-authored-by: Jameson Nash (cherry picked from commit d3ee213987986d5a5acf702061c60a46cb13cab4) --- base/pointer.jl | 1 + test/core.jl | 2 ++ 2 files changed, 3 insertions(+) diff --git a/base/pointer.jl b/base/pointer.jl index 86513c076ade6..2f67c8a6757cb 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -64,6 +64,7 @@ unsafe_convert(::Type{Ptr{Int8}}, s::String) = ccall(:jl_string_ptr, Ptr{Int8}, cconvert(::Type{<:Ptr}, a::Array) = getfield(a, :ref) unsafe_convert(::Type{Ptr{S}}, a::AbstractArray{T}) where {S,T} = convert(Ptr{S}, unsafe_convert(Ptr{T}, a)) +unsafe_convert(::Type{Ptr{T}}, a::Array{T}) where {T} = unsafe_convert(Ptr{T}, a.ref) unsafe_convert(::Type{Ptr{T}}, a::AbstractArray{T}) where {T} = error("conversion to pointer not defined for $(typeof(a))") # TODO: add this deprecation to give a better error: # cconvert(::Type{<:Ptr}, a::AbstractArray) = error("conversion to pointer not defined for $(typeof(a))") diff --git a/test/core.jl b/test/core.jl index aa16380c8a866..0eca2251cd146 100644 --- a/test/core.jl +++ b/test/core.jl @@ -8127,3 +8127,5 @@ let M = @__MODULE__ @test Core.set_binding_type!(M, :a_typed_global) === nothing @test Core.get_binding_type(M, :a_typed_global) === Tuple{Union{Integer,Nothing}} end + +@test Base.unsafe_convert(Ptr{Int}, [1]) !== C_NULL From d98f9cdec9a7a4e3d38e3000d5dd66d6e8c1a33b Mon Sep 17 00:00:00 2001 From: inky Date: Tue, 5 Mar 2024 04:00:14 -0600 Subject: [PATCH 17/55] build: remove extra .a file (#53596) fix #53569 (cherry picked from commit b50344fa0a43bf1da7855a3dcd4dacaae65462bb) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 89fe35b470094..5ce3003dba231 100644 --- a/Makefile +++ b/Makefile @@ -289,6 +289,7 @@ else ifeq ($(JULIA_BUILD_MODE),debug) -$(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/ endif -$(INSTALL_M) $(wildcard $(build_private_libdir)/*.a) $(DESTDIR)$(private_libdir)/ + -rm -f $(DESTDIR)$(private_libdir)/sys-o.a # We have a single exception; we want 7z.dll to live in private_libexecdir, # not bindir, so that 7z.exe can find it. From fb60a88cc9a633e1b17207b275520c11746c16f1 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 5 Mar 2024 18:23:08 +0100 Subject: [PATCH 18/55] fix error path in `precompilepkgs` (#53606) this was accidentally left when porting this from Pkg.jl (cherry picked from commit 427da5c38ee08ab8477f2cd706c605d2d0bcb84c) --- base/precompilation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/precompilation.jl b/base/precompilation.jl index 4efd35836a29d..9eaa0539f522e 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -436,7 +436,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false # TODO: actually handle packages from other envs in the stack return else - error("No direct dependencies outside of the sysimage found matching $(repr([p.name for p in pkgs]))") + error("No direct dependencies outside of the sysimage found matching $(pkgs)") end end From 2d89fee87020d1d5335be00951d2626ea94a447a Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Tue, 5 Mar 2024 16:32:17 -0500 Subject: [PATCH 19/55] bump Pkg --- .../Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 | 1 - .../Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 | 1 - .../Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/md5 | 1 + .../Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/md5 create mode 100644 deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/sha512 diff --git a/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 b/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 deleted file mode 100644 index 279472ca7aa3d..0000000000000 --- a/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -777fe3e3dd5b6c7c1b436d3077d91c13 diff --git a/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 b/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 deleted file mode 100644 index 5e26fbd8577fa..0000000000000 --- a/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -b832093b4e387460a1d96526169e12c6ee2ff5a8bd1961c362dbcf4f4839790f3a0bdcf1d0e1524b77eea662cc7590483fc7e4e674f94f5e7f291970778ab128 diff --git a/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/md5 b/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/md5 new file mode 100644 index 0000000000000..96ceb05a2c8d4 --- /dev/null +++ b/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/md5 @@ -0,0 +1 @@ +0d226d1a3cf5c4cf207788fc3c5ba50c diff --git a/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/sha512 b/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/sha512 new file mode 100644 index 0000000000000..304829ca32df0 --- /dev/null +++ b/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/sha512 @@ -0,0 +1 @@ +e9516466573c24d2bd90645edbdc9c2dbe3f4f4e1704d665525429ff6a463208ff849ebb7c4f9d2d2627d69d1b2cc56a9b305a89e30a0e16b666cbd91f34ced3 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 048b652a0a272..9ca239a2c6b5a 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 76070d295fc4a1f27f852e05400bbc956962e084 +PKG_SHA1 = e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From f46f0b86c672ff2058df4e81d6e26440b2d634b4 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Wed, 28 Feb 2024 18:06:12 -0500 Subject: [PATCH 20/55] task splitting: change additive accumulation to multiplicative (#53408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue raised [here](https://discourse.julialang.org/t/linear-relationship-between-xoshiro-tasks/110454). Given this definition, look at output values: ```jl using .Threads macro fetch(ex) :(fetch(@spawn($(esc(ex))))) end function taskCorrelatedXoshiros(+ = +) r11, r10 = @fetch (@fetch(rand(UInt64)), rand(UInt64)) r01, r00 = (@fetch(rand(UInt64)), rand(UInt64)) r01 + r10 - (r00 + r11) end ``` Before: ```jl julia> sort!(unique(taskCorrelatedXoshiros() for _ = 1:1000)) 9-element Vector{UInt64}: 0x0000000000000000 0x0000000000000001 0x00000000007fffff 0x0000000000800000 0x0000000000800001 0xffffffffff7fffff 0xffffffffff800000 0xffffffffff800001 0xffffffffffffffff ``` After: ```jl julia> sort!(unique(taskCorrelatedXoshiros() for _ = 1:1000)) 1000-element Vector{UInt64}: 0x000420647a085960 0x0038c5b889b585c6 0x007b29fae8107ff7 0x00e73b9e883ac1c8 ⋮ 0xfe68be9c0dde1e88 0xfeca042354218c35 0xfeeb8203e470c96b 0xfff5dbb8771307b9 julia> sort!(unique(taskCorrelatedXoshiros(*) for _ = 1:1000)) 1000-element Vector{UInt64}: 0x00126f951c1e56dc 0x0025a82477ffac08 0x002dd82c9986457a 0x00a713c4d56a3dbc ⋮ 0xfe2e40a5b345095e 0xfe77b90881967436 0xfea2559be63f1701 0xff88b5b28cefac5f ``` (cherry picked from commit 77c06727559c3dbb87e5fde46cec87ce59ba61c8) --- src/task.c | 341 ++++++++++++++++++++++------------------------------- 1 file changed, 143 insertions(+), 198 deletions(-) diff --git a/src/task.c b/src/task.c index 4bf12250b7d2c..34e68e738abae 100644 --- a/src/task.c +++ b/src/task.c @@ -856,202 +856,143 @@ guaranteed to avoid collisions between the RNG streams of all tasks. The main RNG is the xoshiro256++ RNG whose state is stored in rngState[0..3]. There is also a small internal RNG used for task forking stored in rngState[4]. This state is used to iterate a linear congruential generator (LCG), which is then -put through four different variations of the strongest PCG output function, -referred to as PCG-RXS-M-XS-64 [1]. This output function is invertible: it maps -a 64-bit state to 64-bit output. This is one of the reasons it's not recommended -for general purpose RNGs unless space is at an absolute premium, but in our -usage invertibility is actually a benefit (as is explained below) and adding as -little additional memory overhead to each task object as possible is preferred. +combined with xoshiro256's state and put through four different variations of +the strongest PCG output function, referred to as PCG-RXS-M-XS-64 [1]. The goal of jl_rng_split is to perturb the state of each child task's RNG in -such a way each that for an entire tree of tasks spawned starting with a given -state in a root task, no two tasks have the same RNG state. Moreover, we want to -do this in a way that is deterministic and repeatable based on (1) the root -task's seed, (2) how many random numbers are generated, and (3) the task tree -structure. The RNG state of a parent task is allowed to affect the initial RNG -state of a child task, but the mere fact that a child was spawned should not -alter the RNG output of the parent. This second requirement rules out using the -main RNG to seed children: if we use the main RNG, we either advance it, which -affects the parent's RNG stream or, if we don't advance it, then every child -would have an identical RNG stream. Therefore some separate state must be -maintained and changed upon forking a child task while leaving the main RNG -state unchanged. - -The basic approach is that used by the DotMix [2] and SplitMix [3] RNG systems: -each task is uniquely identified by a sequence of "pedigree" numbers, indicating -where in the task tree it was spawned. This vector of pedigree coordinates is -then reduced to a single value by computing a dot product with a shared vector -of random weights. The weights are common but each pedigree of each task is -distinct, so the dot product of each task is unlikely to be the same. The DotMix -paper provides a proof that this dot product hash value (referred to as a -"compression function") is collision resistant in the sense the the pairwise -collision probability of two distinct tasks is 1/N where N is the number of -possible weight values. Both DotMix and SplitMix use a prime value of N because -the proof requires that the difference between two distinct pedigree coordinates -have a multiplicative inverse, which is guaranteed by N being prime since all -values are invertible then. We take a somewhat different approach: instead of -assigning n-ary pedigree coordinates, we assign binary tree coordinates to -tasks, which means that our pedigree vectors have only 0/1 and differences -between them can only be -1, 0 or 1. Since the only possible non-zero coordinate -differences are ±1 which are invertible regardless of the modulus, we can use a -modulus of 2^64, which is far easier and more efficient then using a prime -modulus. It also means that when accumulating the dot product incrementally, as -described in SplitMix, we don't need to multiply weights by anything, we simply -add the random weight for the current task tree depth to the parent's dot -product to derive the child's dot product. - -we instead limit pedigree coordinates to being binary, guaranteeing -invertibility regardless of modulus. When a task spawns a child, the parent and -child share the parent's previous pedigree prefix and the parent appends a zero -to its coordinates, which doesn't affect the task's dot product value, while the -child appends a one, which does produce a new dot product. In this manner a -binary pedigree vector uniquely identifies each task and since the coordinates -are binary, the difference between coordinates is always invertible: 1 and -1 -are their own multiplicative inverses regardless of the modulus. - -How does our assignment of pedigree coordinates to tasks differ from DotMix and -SplitMix? In DotMix and SplitMix, each task has a fixed pedigree vector that -never changes. The root tasks's pedigree is `()`, its first child's pedigree is -`(0,)`, its second child's pedigree is `(2,)` and so on. The length of a task's -pedigree tuple corresponds to how many ancestors tasks it has. Our approach -instead appends 0 to the parent's pedigree when it forks a child and appends 1 -to the child's pedigree at the same time. The root task starts with a pedigree -of `()` as before, but when it spawns a child, we update its pedigree to `(0,)` -and give its child a pedigree of `(1,)`. When the root task then spawns a second -child, we update its pedigree to `(0,0)` and give it's second child a pedigree -of `(0,1)`. If the first child spawns a grandchild, the child's pedigree is -changed from `(1,)` to `(1,0)` and the grandchild is assigned a pedigree of -`(1,1)`. In other words, DotMix and SplitMix build an n-ary tree where every -node is a task: parent nodes are higher up the tree and child tasks are children -in the pedigree tree. Our approach is to build a binary tree where only leaves -are tasks and each task spawn replaces a leaf in the tree with two leaves: the -parent moves to the left/zero leaf while the child is the right/one leaf. Since -the tree is binary, the pedigree coordinates are binary. - -It may seem odd for a task's pedigree coordinates to change, but note that we -only ever append zeros to a task's pedigree, which does not change its dot -product. So while the pedigree changes, the dot product is fixed. What purpose -does appending zeros like this serve if the task's dot product doesn't change? -Changing the pedigree length (which is also the binary tree depth) ensures that -the next child spawned by that task will have new and different dot product from -the previous child since it will have a different pseudo-random weight added to -the parent's dot product value. Whereas the pedigree length in DotMix and -SplitMix is unchanging and corresponds to how many ancestors a task has, in our -scheme the pedigree length corresponds to the number of ancestors *plus* -children a task has, which increases every time it spawns another child. - -We use the LCG in rngState[4] to generate pseudorandom weights for the dot -product. Each time a child is forked, we update the LCG in both parent and child -tasks. In the parent, that's all we have to do -- the main RNG state remains -unchanged. (Recall that spawning a child should *not* affect subsequent RNG -draws in the parent). The next time the parent forks a child, the dot product -weight used will be different, corresponding to being a level deeper in the -pedigree tree. In the child, we use the LCG state to generate four pseudorandom -64-bit weights (more below) and add each weight to one of the xoshiro256 state -registers, rngState[0..3]. If we assume the main RNG remains unused in all -tasks, then each register rngState[0..3] accumulates a different dot product -hash as additional child tasks are spawned. Each one is collision resistant with -a pairwise collision chance of only 1/2^64. Assuming that the four pseudorandom -64-bit weight streams are sufficiently independent, the pairwise collision -probability for distinct tasks is 1/2^256. If we somehow managed to spawn a -trillion tasks, the probability of a collision would be on the order of 1/10^54. -In other words, practically impossible. Put another way, this is the same as the -probability of two SHA256 hash values accidentally colliding, which we generally -consider so unlikely as not to be worth worrying about. - -What about the random "junk" that's in the xoshiro256 state registers from -normal use of the RNG? For a tree of tasks spawned with no intervening samples -taken from the main RNG, all tasks start with the same junk which doesn't affect -the chance of collision. The Dot/SplitMix papers even suggest adding a random -base value to the dot product, so we can consider whatever happens to be in the -xoshiro256 registers to be that. What if the main RNG gets used between task -forks? In that case, the initial state registers will be different. The DotMix -collision resistance proof doesn't apply without modification, but we can -generalize the setup by adding a different base constant to each compression -function and observe that we still have a 1/N chance of the weight value -matching that exact difference. This proves collision resistance even between -tasks whose dot product hashes are computed with arbitrary offsets. We can -conclude that this scheme provides collision resistance even in the face of -different starting states of the main RNG. Does this seem too good to be true? -Perhaps another way of thinking about it will help. Suppose we seeded each task -completely randomly. Then there would also be a 1/2^256 chance of collision, -just as the DotMix proof gives. Essentially what the proof is telling us is that -if the weights are chosen uniformly and uncorrelated with the rest of the -compression function, then the dot product construction is a good enough way to -pseudorandomly seed each task based on its parent's RNG state and where in the -task tree it lives. From that perspective, all we need to believe is that the -dot product construction is random enough (assuming the weights are), and it -becomes easier to believe that adding an arbitrary constant to each dot product -value doesn't make its randomness any worse. - -This leaves us with the question of how to generate four pseudorandom weights to -add to the rngState[0..3] registers at each depth of the task tree. The scheme -used here is that a single 64-bit LCG state is iterated in both parent and child -at each task fork, and four different variations of the PCG-RXS-M-XS-64 output -function are applied to that state to generate four different pseudorandom -weights. Another obvious way to generate four weights would be to iterate the -LCG four times per task split. There are two main reasons we've chosen to use -four output variants instead: - -1. Advancing four times per fork reduces the set of possible weights that each - register can be perturbed by from 2^64 to 2^60. Since collision resistance is - proportional to the number of possible weight values, that would reduce - collision resistance. While it would still be strong engough, why reduce it? - -2. It's easier to compute four PCG output variants in parallel. Iterating the - LCG is inherently sequential. PCG variants can be computed independently. All - four can even be computed at once with SIMD vector instructions. The C - compiler doesn't currently choose to do that transformation, but it could. - -A key question is whether the approach of using four variations of PCG-RXS-M-XS -is sufficiently random both within and between streams to provide the collision -resistance we expect. We obviously can't test that with 256 bits, but we have -tested it with a reduced state analogue using four PCG-RXS-M-XS-8 output -variations applied to a common 8-bit LCG. Test results do indicate sufficient -independence: a single register has collisions at 2^5 while four registers only -start having collisions at 2^20. This is actually better scaling of collision -resistance than we theoretically expect. In theory, with one byte of resistance -we have a 50% chance of some collision at 20 tasks, which matches what we see, -but four bytes should give a 50% chance of collision at 2^17 tasks and our -reduced size analogue construction remains collision free at 2^19 tasks. This -may be due to the next observation, which is that the way we generate -pseudorandom weights actually guarantees collision avoidance in many common -situations rather than merely providing collision resistance and thus is better -than true randomness. - -In the specific case where a parent task spawns a sequence of child tasks with -no intervening usage of its main RNG, the parent and child tasks are actually -_guaranteed_ to have different RNG states. This is true because the four PCG -streams each produce every possible 2^64 bit output exactly once in the full -2^64 period of the LCG generator. This is considered a weakness of PCG-RXS-M-XS -when used as a general purpose RNG, but is quite beneficial in this application. -Since each of up to 2^64 children will be perturbed by different weights, they -cannot have hash collisions. What about parent colliding with child? That can -only happen if all four main RNG registers are perturbed by exactly zero. This -seems unlikely, but could it occur? Consider the core of the output function: - - p ^= p >> ((p >> 59) + 5); - p *= m[i]; - p ^= p >> 43 - -It's easy to check that this maps zero to zero. An unchanged parent RNG can only -happen if all four `p` values are zero at the end of this, which implies that -they were all zero at the beginning. However, that is impossible since the four -`p` values differ from `x` by different additive constants, so they cannot all -be zero. Stated more generally, this non-collision property: assuming the main -RNG isn't used between task forks, sibling and parent tasks cannot have RNG -collisions. If the task tree structure is more deeply nested or if there are -intervening uses of the main RNG, we're back to relying on "merely" 256 bits of -collision resistance, but it's nice to know that in what is likely the most -common case, RNG collisions are actually impossible. This fact may also explain -better-than-theoretical collision resistance observed in our experiment with a -reduced size analogue of our hashing system. +such a way that for an entire tree of tasks spawned starting with a given root +task state, no two tasks have the same RNG state. Moreover, we want to do this +in a way that is deterministic and repeatable based on (1) the root task's seed, +(2) how many random numbers are generated, and (3) the task tree structure. The +RNG state of a parent task is allowed to affect the initial RNG state of a child +task, but the mere fact that a child was spawned should not alter the RNG output +of the parent. This second requirement rules out using the main RNG to seed +children: if we use the main RNG, we either advance it, which affects the +parent's RNG stream or, if we don't advance it, then every child would have an +identical RNG stream. Therefore some separate state must be maintained and +changed upon forking a child task while leaving the main RNG state unchanged. + +The basic approach is a generalization and simplification of that used in the +DotMix [2] and SplitMix [3] RNG systems: each task is uniquely identified by a +sequence of "pedigree" numbers, indicating where in the task tree it was +spawned. This vector of pedigree coordinates is then reduced to a single value +by computing a "dot product" with a shared vector of random weights. I write +"dot product" in quotes because what we use is not an actual dot product. The +linear dot product construction used in both DotMix and SplitMix was found by +@foobar_iv2 [4] to allow easy construction of linear relationships between the +main RNG states of tasks, which was in turn reflected in observable linear +relationships between the outputs of their RNGs. This relationship was between a +minimum of four tasks, so doesn't constitute a collision, per se, but is clearly +undesirable and highlights a hazard of the plain dot product construction. + +As in DotMix and SplitMix, each task is assigned unique task "pedigree" +coordinates. Our pedigree construction is a bit different and uses only binary +coordinates rather than arbitrary integers. Each pedigree is an infinite +sequence of ones and zeros with only finitely many ones. Each task has a "fork +index": the root task has index 0; the fork index of a task the jth child task +of a parent task with fork index i is i+j. The root task's coordinates are all +zeros; each child task's coordinates are the same as its parents except at its +fork index, where the parent has a zero while the child has a one. A task's +coordinates after its fork index are all zeros. The coordinates of a tasks +ancestors are all prefixes of its own coordinates, padded with zeros. + +Also as in DotMix and SplitMix, we generate a sequence of pseudorandom weights +to combine with the coordinates of each task. This sequence is common across all +tasks, and different mix values for each task derive from their coordinates +being different. In DotMix and SplitMix, this is a literal dot product: the +pseudorandom weights are multiplied by corresponding task coordinate and added +up. While this does provably make collisions as unlikely as randomly assigned +task seeds, this linear construction can be used to create linearly correlated +states between tasks. However, it turns out that the compression construction +need not be linear, commutative, associative, etc. which allows us to avoid any +linear or other obvious correlations between related sets of tasks. + +To generalize SplitMix's optimized dot product construction, we similarly +compute each task's compression function value incrementally by combining the +parent's compression value with pseudorandom weight corresponding with the +child's fork index. Formally, if the parent's compression value is c then we can +compute the child's compression value as c′ = f(c, wᵢ) where w is the vector of +pseudorandom weights. What is f? It can be any function that is bijective in +each argument for all values of the other argument: + + * For all c: w ↦ f(c, w) is bijective + * For all w: c ↦ f(c, w) is bijective + +The proof that these requirements are sufficient to ensure collision resistance +is in the linked discussion [4]. DotMix/SplitMix are a special case where f is +just addition. Instead we use a much less simple mixing function: + + 1. We use (2c+1)(2w+1)÷2 % 2^64 to mix the bits of c and w + 2. We then apply the PCG-RXS-M-XS-64 output function + +The first step thoroughly mixes the bits of the previous compression value and +the pseudorandom weight value using multiplication, which is non-commutative +with xoshiro's operations (xor, shift, rotate). This mixing function is a +bijection on each argument witnessed by these inverses: + + * c′ ↦ (2c′+1)(2w+1)⁻¹÷2 % 2^64 + * w′ ↦ (2c+1)⁻¹(2w′+1)÷2 % 2^64 + +The second PCG output step is a bijection and designed to be significantly +non-linear -- non-linear enough to mask the linearity of the LCG that drives the +PCG-RXS-M-XS-64 RNG and allows it to pass statistical RNG test suites despite +having the same size state and output. In particular, since this mixing function +is highly non-associative and non-linear, we (hopefully) don't have any +discernible relationship between these values: + + * c₀₀ = c + * c₁₀ = f(c, wᵢ) + * c₀₁ = f(c, wⱼ) + * c₁₁ = f(f(c, wᵢ), wⱼ) + +When f is simply `+` then these have a very obvious relationship: + + c₀₀ + c₁₁ == c₁₀ + c₀₁ + +This relationship holds regardless of what wᵢ and wⱼ are and is precisely what +allows easy creation of correlated tasks with the DotMix/SplitMix construction +that we previously used. Expressing any relationship between these values with +our mixing function would require inverting the PCG output function (doable but +non-trivial), knowing the weights wᵢ and wⱼ, and then applying the inversion +functions for those weights appropriately. Since the weights are pseudo-randomly +generated and not directly observable, this is infeasible. + +We maintain an LCG in rngState[4] to generate pseudorandom weights. An LCG by +itself is a very bad RNG, but we combine this one with xoshiro256 state +registers in a non-trivial way and then apply the PCG-RXS-M-XS-64 output +function to that. Even if the xoshiro256 states are all zeros, which they should +never be, the output would be the same as PCG-RXS-M-XS-64, which is a solid +statistical RNG. + +Each time a child is forked, we update the LCG in both parent and child tasks, +corresponding to increasing the fork index. In the parent, that's all we have to +do -- the main RNG state remains unchanged. Recall that spawning a child should +*not* affect subsequent RNG draws in the parent. The next time the parent forks +a child, the mixing weight used will be different. In the child, we use the LCG +state to perturb the child's main RNG state registers, rngState[0..3]. + +Since we want these registers to behave independently, we use four different +variations on f to mix the LCG state with each of the four main RNG registers. +Each variation first xors the LCG state with a different random constant before +combining that value above with the old register state via multiplication; the +PCG-RXS-M-XS-64 output function is then applied to that mixed state, with a +different multiplier constant for each variation / register index. Xor is used +in the first step since we multiply the result with the state immediately after +and multiplication distributes over `+` and commutes with `*`, which makes both +options suspect; multiplication doesn't distribute over or commute with xor. We +also use a different odd multiplier in PCG-RXS-M-XS-64 for each RNG register. +These three sources of variation (different xor constants, different xoshiro256 +state, different PCG multipliers) are sufficient for each of the four outputs to +behave statistically independently. [1]: https://www.pcg-random.org/pdf/hmc-cs-2014-0905.pdf [2]: http://supertech.csail.mit.edu/papers/dprng.pdf [3]: https://gee.cs.oswego.edu/dl/papers/oopsla14.pdf + +[4]: +https://discourse.julialang.org/t/linear-relationship-between-xoshiro-tasks/110454 */ void jl_rng_split(uint64_t dst[JL_RNG_SIZE], uint64_t src[JL_RNG_SIZE]) JL_NOTSAFEPOINT { @@ -1060,26 +1001,30 @@ void jl_rng_split(uint64_t dst[JL_RNG_SIZE], uint64_t src[JL_RNG_SIZE]) JL_NOTSA src[4] = dst[4] = x * 0xd1342543de82ef95 + 1; // high spectrum multiplier from https://arxiv.org/abs/2001.05304 + // random xor constants static const uint64_t a[4] = { - 0xe5f8fa077b92a8a8, // random additive offsets... - 0x7a0cd918958c124d, - 0x86222f7d388588d4, - 0xd30cbd35f2b64f52 + 0x214c146c88e47cb7, + 0xa66d8cc21285aafa, + 0x68c7ef2d7b1a54d4, + 0xb053a7d7aa238c61 }; + // random odd multipliers static const uint64_t m[4] = { 0xaef17502108ef2d9, // standard PCG multiplier - 0xf34026eeb86766af, // random odd multipliers... + 0xf34026eeb86766af, 0x38fd70ad58dd9fbb, 0x6677f9b93ab0c04d }; // PCG-RXS-M-XS-64 output with four variants for (int i = 0; i < 4; i++) { - uint64_t p = x + a[i]; - p ^= p >> ((p >> 59) + 5); - p *= m[i]; - p ^= p >> 43; - dst[i] = src[i] + p; // SplitMix dot product + uint64_t c = src[i]; + uint64_t w = x ^ a[i]; + c += w*(2*c + 1); // c = (2c+1)(2w+1)÷2 % 2^64 (double bijection) + c ^= c >> ((c >> 59) + 5); + c *= m[i]; + c ^= c >> 43; + dst[i] = c; } } From 2d485dc0ce2c455b7d3d3c7e0bf0a6a7afc878e2 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 2 Mar 2024 11:49:08 -0500 Subject: [PATCH 21/55] revert "Add @create_log_macro for making custom styled logging macros (#52196)" (#53551) (cherry picked from commit 8bf6a073a2bec5c015a8da3a8d5bb22d129d15ca) --- NEWS.md | 3 --- base/logging.jl | 18 ++++++--------- stdlib/Logging/docs/src/index.md | 14 +----------- stdlib/Logging/src/ConsoleLogger.jl | 1 - stdlib/Logging/src/Logging.jl | 35 ----------------------------- stdlib/Logging/test/runtests.jl | 20 +++++------------ 6 files changed, 14 insertions(+), 77 deletions(-) diff --git a/NEWS.md b/NEWS.md index 42d36b606e839..9156d3de63830 100644 --- a/NEWS.md +++ b/NEWS.md @@ -149,9 +149,6 @@ Standard library changes * `lu` and `issuccess(::LU)` now accept an `allowsingular` keyword argument. When set to `true`, a valid factorization with rank-deficient U factor will be treated as success instead of throwing an error. Such factorizations are now shown by printing the factors together with a "rank-deficient" note rather than printing a "Failed Factorization" message ([#52957]). #### Logging -* New `@create_log_macro` macro for creating new log macros like `@info`, `@warn` etc. For instance - `@create_log_macro MyLog 1500 :magenta` will create `@mylog` to be used like `@mylog "hello"` which - will show as `┌ MyLog: hello` etc. ([#52196]) #### Printf diff --git a/base/logging.jl b/base/logging.jl index d0f612c31eeae..bef8a89118371 100644 --- a/base/logging.jl +++ b/base/logging.jl @@ -172,18 +172,14 @@ const AboveMaxLevel = LogLevel( 1000001) # Global log limiting mechanism for super fast but inflexible global log limiting. const _min_enabled_level = Ref{LogLevel}(Debug) -# stored as LogLevel => (name, color) -const custom_log_levels = Dict{LogLevel,Tuple{Symbol,Union{Symbol,Int}}}() - function show(io::IO, level::LogLevel) - if haskey(custom_log_levels, level) print(io, custom_log_levels[level][1]) - elseif level == BelowMinLevel print(io, "BelowMinLevel") - elseif level == Debug print(io, "Debug") - elseif level == Info print(io, "Info") - elseif level == Warn print(io, "Warn") - elseif level == Error print(io, "Error") - elseif level == AboveMaxLevel print(io, "AboveMaxLevel") - else print(io, "LogLevel($(level.level))") + if level == BelowMinLevel print(io, "BelowMinLevel") + elseif level == Debug print(io, "Debug") + elseif level == Info print(io, "Info") + elseif level == Warn print(io, "Warn") + elseif level == Error print(io, "Error") + elseif level == AboveMaxLevel print(io, "AboveMaxLevel") + else print(io, "LogLevel($(level.level))") end end diff --git a/stdlib/Logging/docs/src/index.md b/stdlib/Logging/docs/src/index.md index 6d0af3ac21321..c2bde11720f4c 100644 --- a/stdlib/Logging/docs/src/index.md +++ b/stdlib/Logging/docs/src/index.md @@ -62,7 +62,7 @@ automatically extracted. Let's examine the user-defined data first: * The *log level* is a broad category for the message that is used for early filtering. There are several standard levels of type [`LogLevel`](@ref); user-defined levels are also possible. - Each built-in log level is distinct in purpose: + Each is distinct in purpose: - [`Logging.Debug`](@ref) (log level -1000) is information intended for the developer of the program. These events are disabled by default. - [`Logging.Info`](@ref) (log level 0) is for general information to the user. @@ -74,17 +74,6 @@ automatically extracted. Let's examine the user-defined data first: Often this log-level is unneeded as throwing an exception can convey all the required information. - You can create logging macros for custom log levels. For instance: - ```julia-repl - julia> using Logging - - julia> @create_log_macro MyLog 200 :magenta - @mylog (macro with 1 method) - - julia> @mylog "hello" - [ MyLog: hello - ``` - * The *message* is an object describing the event. By convention `AbstractString`s passed as messages are assumed to be in markdown format. Other types will be displayed using `print(io, obj)` or `string(obj)` for @@ -315,7 +304,6 @@ Logging.Warn Logging.Error Logging.BelowMinLevel Logging.AboveMaxLevel -Logging.@create_log_macro ``` ### [Processing events with AbstractLogger](@id AbstractLogger-interface) diff --git a/stdlib/Logging/src/ConsoleLogger.jl b/stdlib/Logging/src/ConsoleLogger.jl index 2950ad1b8cf95..08e4a8c6b2efe 100644 --- a/stdlib/Logging/src/ConsoleLogger.jl +++ b/stdlib/Logging/src/ConsoleLogger.jl @@ -58,7 +58,6 @@ end showvalue(io, ex::Exception) = showerror(io, ex) function default_logcolor(level::LogLevel) - level in keys(custom_log_levels) ? custom_log_levels[level][2] : level < Info ? :log_debug : level < Warn ? :log_info : level < Error ? :log_warn : diff --git a/stdlib/Logging/src/Logging.jl b/stdlib/Logging/src/Logging.jl index c05d3b7227c34..3822bde2e630b 100644 --- a/stdlib/Logging/src/Logging.jl +++ b/stdlib/Logging/src/Logging.jl @@ -23,7 +23,6 @@ for sym in [ Symbol("@warn"), Symbol("@error"), Symbol("@logmsg"), - :custom_log_levels, :with_logger, :current_logger, :global_logger, @@ -32,39 +31,6 @@ for sym in [ @eval const $sym = Base.CoreLogging.$sym end -""" - @create_log_macro(name::Symbol, level::Int, face::Union{Symbol, StyledStrings.Face}) - -Creates a custom log macro like `@info`, `@warn` etc. with a given `name`, -`level` to be displayed with `face`. The macro created is named with the -lowercase form of `name` but the given form is used for the printing. - -```julia-repl -julia> @create_log_macro(:MyLog, 200, :magenta) -@mylog (macro with 1 method) - -julia> @mylog "hello" -[ MyLog: hello -``` -""" -macro create_log_macro(name, level, color) - macro_name = Symbol(lowercase(string(name))) - macro_string = QuoteNode(name) - loglevel = LogLevel(level) - if loglevel in (BelowMinLevel, Debug, Info, Warn, Error, AboveMaxLevel) - throw(ArgumentError("Cannot use the same log level as a built in log macro")) - end - if haskey(custom_log_levels, loglevel) - throw(ArgumentError("Custom log macro already exists for given log level")) - end - quote - $(custom_log_levels)[$(esc(loglevel))] = ($(macro_string), $(esc(color))) - macro $(esc(macro_name))(exs...) - $(Base.CoreLogging.logmsg_code)(($(Base.CoreLogging.@_sourceinfo))..., $(esc(loglevel)), exs...) - end - end -end - # LogLevel aliases (re-)documented here (JuliaLang/julia#40978) """ Debug @@ -115,7 +81,6 @@ export @warn, @error, @logmsg, - @create_log_macro, with_logger, current_logger, global_logger, diff --git a/stdlib/Logging/test/runtests.jl b/stdlib/Logging/test/runtests.jl index 57866ff0a18d2..a244facee3468 100644 --- a/stdlib/Logging/test/runtests.jl +++ b/stdlib/Logging/test/runtests.jl @@ -7,8 +7,8 @@ import Logging: min_enabled_level, shouldlog, handle_message @noinline func1() = backtrace() # see "custom log macro" testset -@create_log_macro CustomLog1 -500 :magenta -@create_log_macro CustomLog2 1500 1 +CustomLog = LogLevel(-500) +macro customlog(exs...) Base.CoreLogging.logmsg_code((Base.CoreLogging.@_sourceinfo)..., esc(CustomLog), exs...) end @testset "Logging" begin @@ -289,24 +289,16 @@ end end @testset "custom log macro" begin - llevel = LogLevel(-500) - - @test_logs (llevel, "foo") min_level=llevel @customlog1 "foo" + @test_logs (CustomLog, "a") min_level=CustomLog @customlog "a" buf = IOBuffer() io = IOContext(buf, :displaysize=>(30,80), :color=>false) - logger = ConsoleLogger(io, llevel) - - with_logger(logger) do - @customlog1 "foo" - end - @test occursin("CustomLog1: foo", String(take!(buf))) - + logger = ConsoleLogger(io, CustomLog) with_logger(logger) do - @customlog2 "hello" + @customlog "a" end - @test occursin("CustomLog2: hello", String(take!(buf))) + @test occursin("LogLevel(-500): a", String(take!(buf))) end @testset "Docstrings" begin From f859acd897596c3646fcb56ef3eb23084b616e98 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 6 Mar 2024 10:04:22 -0500 Subject: [PATCH 22/55] Unexport with, at_with, and ScopedValue from Base (#53004) fixes #52535 --------- Co-authored-by: Kristoffer Carlsson (cherry picked from commit 6335386555ff5af3fa7844a8674473871c8427dd) --- base/exports.jl | 5 ----- base/logging.jl | 1 + base/scopedvalues.jl | 2 ++ doc/src/base/scopedvalues.md | 25 ++++++++++++++++++++++--- test/scopedvalues.jl | 10 +++++----- 5 files changed, 30 insertions(+), 13 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index a31ea1b0fa842..ba022082ced35 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -653,11 +653,6 @@ export sprint, summary, -# ScopedValue - with, - @with, - ScopedValue, - # logging @debug, @info, diff --git a/base/logging.jl b/base/logging.jl index bef8a89118371..75b8eea5051d2 100644 --- a/base/logging.jl +++ b/base/logging.jl @@ -3,6 +3,7 @@ module CoreLogging import Base: isless, +, -, convert, show +import Base: ScopedValue, with, @with export AbstractLogger, diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index ee4571b8ae497..339ee717d5fd6 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -19,6 +19,8 @@ Dynamic scopes are propagated across tasks. # Examples ```jldoctest +julia> using Base.ScopedValues; + julia> const sval = ScopedValue(1); julia> sval[] diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index 0de29308c5df8..2991522900e1a 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -17,8 +17,8 @@ concurrently. Scoped values were introduced in Julia 1.11. In Julia 1.8+ a compatible implementation is available from the package ScopedValues.jl. -In its simplest form you can create a [`ScopedValue`](@ref) with a -default value and then use [`with`](@ref Base.with) or [`@with`](@ref) to +In its simplest form you can create a [`Base.ScopedValue`](@ref) with a +default value and then use [`Base.with`](@ref with) or [`Base.@with`](@ref) to enter a new dynamic scope. The new scope will inherit all values from the parent scope @@ -54,6 +54,8 @@ f() # 1 Now using a `ScopedValue` we can use **dynamic** scoping. ```julia +using Base.ScopedValues + x = ScopedValue(1) f() = @show x[] with(x=>5) do @@ -70,6 +72,8 @@ and you can set the value of multiple `ScopedValue`s with one call to `with`. ```julia +using Base.ScopedValues + const scoped_val = ScopedValue(1) const scoped_val2 = ScopedValue(0) @@ -94,6 +98,8 @@ Since `with` requires a closure or a function and creates another call-frame, it can sometimes be beneficial to use the macro form. ```julia +using Base.ScopedValues + const STATE = ScopedValue{State}() with_state(f, state::State) = @with(STATE => state, f()) ``` @@ -106,7 +112,9 @@ The parent task and the two child tasks observe independent values of the same scoped value at the same time. ```julia +using Base.ScopedValues import Base.Threads: @spawn + const scoped_val = ScopedValue(1) @sync begin with(scoped_val => 2) @@ -128,7 +136,9 @@ values. You might want to explicitly [unshare mutable state](@ref unshare_mutabl when entering a new dynamic scope. ```julia +using Base.ScopedValues import Base.Threads: @spawn + const sval_dict = ScopedValue(Dict()) # Example of using a mutable value wrongly @@ -161,6 +171,8 @@ are not well suited for this kind of propagation; our only alternative would hav been to thread a value through the entire call-chain. ```julia +using Base.ScopedValues + const LEVEL = ScopedValue(:GUEST) function serve(request, response) @@ -189,7 +201,9 @@ end ### [Unshare mutable state](@id unshare_mutable_state) ```julia +using Base.ScopedValues import Base.Threads: @spawn + const sval_dict = ScopedValue(Dict()) # If you want to add new values to the dict, instead of replacing @@ -210,6 +224,7 @@ be in (lexical) scope. This means most often you likely want to use scoped value as constant globals. ```julia +using Base.ScopedValues const sval = ScopedValue(1) ``` @@ -218,7 +233,9 @@ Indeed one can think of scoped values as hidden function arguments. This does not preclude their use as non-globals. ```julia +using Base.ScopedValues import Base.Threads: @spawn + function main() role = ScopedValue(:client) @@ -241,6 +258,8 @@ If you find yourself creating many `ScopedValue`'s for one given module, it may be better to use a dedicated struct to hold them. ```julia +using Base.ScopedValues + Base.@kwdef struct Configuration color::Bool = false verbose::Bool = false @@ -260,7 +279,7 @@ end Base.ScopedValues.ScopedValue Base.ScopedValues.with Base.ScopedValues.@with -Base.isassigned(::ScopedValue) +Base.isassigned(::Base.ScopedValues.ScopedValue) Base.ScopedValues.get ``` diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index 39de22bc2bee5..ca700521b50cd 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -1,5 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -import Base: ScopedValues +using Base.ScopedValues @testset "errors" begin @test ScopedValue{Float64}(1)[] == 1.0 @@ -67,13 +67,13 @@ import Base.Threads: @spawn end @testset "show" begin - @test sprint(show, ScopedValue{Int}()) == "ScopedValue{$Int}(undefined)" - @test sprint(show, sval) == "ScopedValue{$Int}(1)" + @test sprint(show, ScopedValue{Int}()) == "Base.ScopedValues.ScopedValue{$Int}(undefined)" + @test sprint(show, sval) == "Base.ScopedValues.ScopedValue{$Int}(1)" @test sprint(show, Core.current_scope()) == "nothing" with(sval => 2.0) do - @test sprint(show, sval) == "ScopedValue{$Int}(2)" + @test sprint(show, sval) == "Base.ScopedValues.ScopedValue{$Int}(2)" objid = sprint(show, Base.objectid(sval)) - @test sprint(show, Core.current_scope()) == "Base.ScopedValues.Scope(ScopedValue{$Int}@$objid => 2)" + @test sprint(show, Core.current_scope()) == "Base.ScopedValues.Scope(Base.ScopedValues.ScopedValue{$Int}@$objid => 2)" end end From dbc02412c8bca0c89df96e99211c090d62ba3b21 Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Thu, 7 Mar 2024 05:55:49 +0800 Subject: [PATCH 23/55] typeintersect: fix `UnionAll` unaliasing bug caused by innervars. (#53553) typeintersect: fix `UnionAll` unaliasing bug caused by innervars. (cherry picked from commit 56f1c8ae62c07cb940e0c4fc02d5dfac9ec73147) --- src/subtype.c | 71 ++++++++++++++++++++++++++++++++++++++----------- test/subtype.jl | 26 ++++++++++++++++++ 2 files changed, 81 insertions(+), 16 deletions(-) diff --git a/src/subtype.c b/src/subtype.c index 4a797aab9831d..8b2c696476061 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -881,10 +881,20 @@ static jl_unionall_t *unalias_unionall(jl_unionall_t *u, jl_stenv_t *e) // in the environment, rename to get a fresh var. JL_GC_PUSH1(&u); while (btemp != NULL) { - if (btemp->var == u->var || - // outer var can only refer to inner var if bounds changed + int aliased = btemp->var == u->var || + // outer var can only refer to inner var if bounds changed (mainly for subtyping path) (btemp->lb != btemp->var->lb && jl_has_typevar(btemp->lb, u->var)) || - (btemp->ub != btemp->var->ub && jl_has_typevar(btemp->ub, u->var))) { + (btemp->ub != btemp->var->ub && jl_has_typevar(btemp->ub, u->var)); + if (!aliased && btemp->innervars != NULL) { + for (size_t i = 0; i < jl_array_len(btemp->innervars); i++) { + jl_tvar_t *ivar = (jl_tvar_t*)jl_array_ptr_ref(btemp->innervars, i); + if (ivar == u->var) { + aliased = 1; + break; + } + } + } + if (aliased) { u = jl_rename_unionall(u); break; } @@ -2839,7 +2849,7 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind // I. Handle indirect innervars (make them behave like direct innervars). // 1) record if btemp->lb/ub has indirect innervars. - // 2) substitute `vb->var` with `varval`/`varval` + // 2) substitute `vb->var` with `varval`/`newvar` // note: We only store the innervar in the outmost `varbinding`, // thus we must check all inner env to ensure the recording/substitution // is complete @@ -2903,6 +2913,7 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind } envind++; } + // FIXME: innervar that depend on `ivar` should also be updated. } } } @@ -3018,7 +3029,8 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind } if (vb->innervars != NULL) { - for (size_t i = 0; i < jl_array_nrows(vb->innervars); i++) { + size_t len = jl_array_nrows(vb->innervars), count = 0; + for (size_t i = 0; i < len; i++) { jl_tvar_t *var = (jl_tvar_t*)jl_array_ptr_ref(vb->innervars, i); // the `btemp->prev` walk is only giving a sort of post-order guarantee (since we are // iterating 2 trees at once), so once we set `wrap`, there might remain other branches @@ -3032,11 +3044,45 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind if (wrap) { if (wrap->innervars == NULL) wrap->innervars = jl_alloc_array_1d(jl_array_any_type, 0); + // FIXME: `var`'s dependence should also be pushed into `wrap->innervars`. jl_array_ptr_1d_push(wrap->innervars, (jl_value_t*)var); + jl_array_ptr_set(vb->innervars, i, (jl_value_t*)NULL); + } + } + for (size_t i = 0; i < len; i++) { + jl_tvar_t *var = (jl_tvar_t*)jl_array_ptr_ref(vb->innervars, i); + if (var) { + if (count < i) + jl_array_ptr_set(vb->innervars, count, (jl_value_t*)var); + count++; + } + } + if (count != len) + jl_array_del_end(vb->innervars, len - count); + if (res != jl_bottom_type) { + while (count > 1) { + int changed = 0; + // Now need to re-sort the vb->innervars using the partial-ordering predicate `jl_has_typevar`. + // If this is slow, we could possibly switch to a simpler graph sort than this triple loop, such as Tarjan's SCC. + // But for now we use a variant on selection sort for partial-orders. + for (size_t i = 0; i < count - 1; i++) { + jl_tvar_t *vari = (jl_tvar_t*)jl_array_ptr_ref(vb->innervars, i); + for (size_t j = i+1; j < count; j++) { + jl_tvar_t *varj = (jl_tvar_t*)jl_array_ptr_ref(vb->innervars, j); + if (jl_has_typevar(varj->lb, vari) || jl_has_typevar(varj->ub, vari)) { + jl_array_ptr_set(vb->innervars, j, (jl_value_t*)vari); + jl_array_ptr_set(vb->innervars, i, (jl_value_t*)varj); + changed = 1; + break; + } + } + if (changed) break; + } + if (!changed) break; } - else if (res != jl_bottom_type) { - if (jl_has_typevar(res, var)) - res = jl_type_unionall((jl_tvar_t*)var, res); + for (size_t i = 0; i < count; i++) { + jl_tvar_t *var = (jl_tvar_t*)jl_array_ptr_ref(vb->innervars, i); + res = jl_type_unionall(var, res); } } } @@ -3056,9 +3102,6 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind static jl_value_t *intersect_unionall_(jl_value_t *t, jl_unionall_t *u, jl_stenv_t *e, int8_t R, int param, jl_varbinding_t *vb) { jl_varbinding_t *btemp = e->vars; - // if the var for this unionall (based on identity) already appears somewhere - // in the environment, rename to get a fresh var. - // TODO: might need to look inside types in btemp->lb and btemp->ub int envsize = 0; while (btemp != NULL) { envsize++; @@ -3066,13 +3109,9 @@ static jl_value_t *intersect_unionall_(jl_value_t *t, jl_unionall_t *u, jl_stenv vb->limited = 1; return t; } - if (btemp->var == u->var || btemp->lb == (jl_value_t*)u->var || - btemp->ub == (jl_value_t*)u->var) { - u = jl_rename_unionall(u); - break; - } btemp = btemp->prev; } + u = unalias_unionall(u, e); JL_GC_PUSH1(&u); vb->var = u->var; e->vars = vb; diff --git a/test/subtype.jl b/test/subtype.jl index 27e8fa604a5e9..7805c232bd050 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -2572,6 +2572,32 @@ let a = Tuple{Union{Nothing, Type{Pair{T1}} where T1}} @test !Base.has_free_typevars(typeintersect(a, b)) end +#issue 53366 +let Y = Tuple{Val{T}, Val{Val{T}}} where T + A = Val{Val{T}} where T + T = TypeVar(:T, UnionAll(A.var, Val{A.var})) + B = UnionAll(T, Val{T}) + X = Tuple{A, B} + @testintersect(X, Y, !Union{}) +end + +#issue 53621 (requires assertions enabled) +abstract type A53621{T, R, C, U} <: AbstractSet{Union{C, U}} end +struct T53621{T, R<:Real, C, U} <: A53621{T, R, C, U} end +let + U = TypeVar(:U) + C = TypeVar(:C) + T = TypeVar(:T) + R = TypeVar(:R) + CC = TypeVar(:CC, Union{C, U}) + UU = TypeVar(:UU, Union{C, U}) + S1 = UnionAll(T, UnionAll(R, Type{UnionAll(C, UnionAll(U, T53621{T, R, C, U}))})) + S2 = UnionAll(C, UnionAll(U, UnionAll(CC, UnionAll(UU, UnionAll(T, UnionAll(R, T53621{T, R, CC, UU})))))) + S = Tuple{S1, S2} + T = Tuple{Type{T53621{T, R}}, AbstractSet{T}} where {T, R} + @testintersect(S, T, !Union{}) +end + #issue 53371 struct T53371{A,B,C,D,E} end S53371{A} = Union{Int, <:A} From 32efb7e7a8bb679fbfa3b7ce7606cd931f78001e Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 7 Mar 2024 15:10:00 +0000 Subject: [PATCH 24/55] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20Pk?= =?UTF-8?q?g=20stdlib=20from=20e7d740ac8=20to=2056c379045?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 87433e53df6818911e0d69dc0a6e0104fdb38321) --- .../Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/md5 | 1 + .../Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/sha512 | 1 + .../Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/md5 | 1 - .../Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/sha512 | 1 - stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/md5 create mode 100644 deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/sha512 diff --git a/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/md5 b/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/md5 new file mode 100644 index 0000000000000..3fc997ee6d795 --- /dev/null +++ b/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/md5 @@ -0,0 +1 @@ +de47c3414e7e3168f77b20676c3f7631 diff --git a/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/sha512 b/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/sha512 new file mode 100644 index 0000000000000..7d14fdc309cfe --- /dev/null +++ b/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/sha512 @@ -0,0 +1 @@ +b4b7ebb7fe636bbb9d653cb826e1a74eadcbc936dbf1a76281a7cc2e439a17f48ea8d04b77d825deb94a0fe228490760df5836837cafbcbd7308251d0be6589f diff --git a/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/md5 b/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/md5 deleted file mode 100644 index 96ceb05a2c8d4..0000000000000 --- a/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -0d226d1a3cf5c4cf207788fc3c5ba50c diff --git a/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/sha512 b/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/sha512 deleted file mode 100644 index 304829ca32df0..0000000000000 --- a/deps/checksums/Pkg-e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -e9516466573c24d2bd90645edbdc9c2dbe3f4f4e1704d665525429ff6a463208ff849ebb7c4f9d2d2627d69d1b2cc56a9b305a89e30a0e16b666cbd91f34ced3 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 9ca239a2c6b5a..a68d4a1697eac 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = e7d740ac82fa4c289bb58f9acb6e60eebb6d8a8e +PKG_SHA1 = 56c379045c8cf876b365c284a5b442f2dc9fc8af PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 2a7100acfbaa209c2f36fdb02fe04395487614dd Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 8 Mar 2024 01:15:35 +0900 Subject: [PATCH 25/55] inlining: use method match signature for union-spliting (#53600) In cases where the results of constant inference, like concrete-eval, are used for union-split inlining, `isa`-blocks are generated using the `result.edge.specTypes` stored within each `result`. However, it's been found that the `edge` returned by abstract interpretation may have been widened by the new `@nospecializeinfer`, which can result in invalid union-splitting. To address this problem, this commit tweaks the inlining algorithm so that it performs union-split inlining using the original signatures that abstract interpretation used for union-split inference, by using `match::MethodMatch`. - fixes #53590 --- base/compiler/ssair/inlining.jl | 23 ++++++++++++----------- test/compiler/inference.jl | 12 ++++++++++++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index d4228cf3c4454..4d46c198d9964 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -1316,11 +1316,11 @@ function handle_any_const_result!(cases::Vector{InliningCase}, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; allow_abstract::Bool, allow_typevars::Bool) if isa(result, ConcreteResult) - return handle_concrete_result!(cases, result, info, state) + return handle_concrete_result!(cases, result, match, info, state) elseif isa(result, SemiConcreteResult) - return handle_semi_concrete_result!(cases, result, info, flag, state; allow_abstract) + return handle_semi_concrete_result!(cases, result, match, info, flag, state; allow_abstract) elseif isa(result, ConstPropResult) - return handle_const_prop_result!(cases, result, info, flag, state; allow_abstract, allow_typevars) + return handle_const_prop_result!(cases, result, match, info, flag, state; allow_abstract, allow_typevars) else @assert result === nothing || result isa VolatileInferenceResult return handle_match!(cases, match, argtypes, info, flag, state; allow_abstract, allow_typevars, volatile_inf_result = result) @@ -1466,11 +1466,11 @@ function handle_match!(cases::Vector{InliningCase}, return true end -function handle_const_prop_result!(cases::Vector{InliningCase}, - result::ConstPropResult, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; +function handle_const_prop_result!(cases::Vector{InliningCase}, result::ConstPropResult, + match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; allow_abstract::Bool, allow_typevars::Bool) mi = result.result.linfo - spec_types = mi.specTypes + spec_types = match.spec_types allow_abstract || isdispatchtuple(spec_types) || return false if !validate_sparams(mi.sparam_vals) (allow_typevars && !may_have_fcalls(mi.def::Method)) || return false @@ -1505,10 +1505,10 @@ function semiconcrete_result_item(result::SemiConcreteResult, end function handle_semi_concrete_result!(cases::Vector{InliningCase}, result::SemiConcreteResult, - @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; - allow_abstract::Bool) + match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; + allow_abstract::Bool) mi = result.mi - spec_types = mi.specTypes + spec_types = match.spec_types allow_abstract || isdispatchtuple(spec_types) || return false validate_sparams(mi.sparam_vals) || return false item = semiconcrete_result_item(result, info, flag, state) @@ -1517,10 +1517,11 @@ function handle_semi_concrete_result!(cases::Vector{InliningCase}, result::SemiC return true end -function handle_concrete_result!(cases::Vector{InliningCase}, result::ConcreteResult, @nospecialize(info::CallInfo), state::InliningState) +function handle_concrete_result!(cases::Vector{InliningCase}, result::ConcreteResult, + match::MethodMatch, @nospecialize(info::CallInfo), state::InliningState) case = concrete_result_item(result, info, state) case === nothing && return false - push!(cases, InliningCase(result.mi.specTypes, case)) + push!(cases, InliningCase(match.spec_types, case)) return true end diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index 65729ab01cb9f..243566596954e 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -5633,3 +5633,15 @@ end # Issue #52613 @test (code_typed((Any,)) do x; TypeVar(x...); end)[1][2] === TypeVar + +# https://github.com/JuliaLang/julia/issues/53590 +func53590(b) = b ? Int : Float64 +function issue53590(b1, b2) + T1 = func53590(b1) + T2 = func53590(b2) + return typejoin(T1, T2) +end +@test issue53590(true, true) == Int +@test issue53590(true, false) == Real +@test issue53590(false, false) == Float64 +@test issue53590(false, true) == Real From 814257c7f3868762bda8b7e68496a39eca23aeb6 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:42:49 +0900 Subject: [PATCH 26/55] add `construct_[post]domtree(::[IRCode|CFG])` interfaces (#53638) We currently have `construct_[post]domtree(::Vector{BasicBlock})` only. The higher level interfaces are often convenient. --- base/compiler/inferencestate.jl | 6 +++--- base/compiler/optimize.jl | 4 ++-- base/compiler/ssair/ir.jl | 6 ++++++ test/compiler/irpasses.jl | 4 ++-- test/compiler/ssair.jl | 14 +++++++------- test/compiler/tarjan.jl | 2 +- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/base/compiler/inferencestate.jl b/base/compiler/inferencestate.jl index 5c1fea2e78e9f..f08fbac8af360 100644 --- a/base/compiler/inferencestate.jl +++ b/base/compiler/inferencestate.jl @@ -177,7 +177,7 @@ mutable struct LazyCFGReachability end function get!(x::LazyCFGReachability) isdefined(x, :reachability) && return x.reachability - domtree = construct_domtree(x.ir.cfg.blocks) + domtree = construct_domtree(x.ir) return x.reachability = CFGReachability(x.ir.cfg, domtree) end @@ -189,8 +189,8 @@ end function get!(x::LazyGenericDomtree{IsPostDom}) where {IsPostDom} isdefined(x, :domtree) && return x.domtree return @timeit "domtree 2" x.domtree = IsPostDom ? - construct_postdomtree(x.ir.cfg.blocks) : - construct_domtree(x.ir.cfg.blocks) + construct_postdomtree(x.ir) : + construct_domtree(x.ir) end const LazyDomtree = LazyGenericDomtree{false} diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index f6d98fa53dede..9b6fcb760adb4 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -578,7 +578,7 @@ function get!(lazyagdomtree::LazyAugmentedDomtree) cfg_insert_edge!(cfg, bb, length(cfg.blocks)) end end - domtree = construct_domtree(cfg.blocks) + domtree = construct_domtree(cfg) return lazyagdomtree.agdomtree = AugmentedDomtree(cfg, domtree) end @@ -1187,7 +1187,7 @@ function slot2reg(ir::IRCode, ci::CodeInfo, sv::OptimizationState) # need `ci` for the slot metadata, IR for the code svdef = sv.linfo.def nargs = isa(svdef, Method) ? Int(svdef.nargs) : 0 - @timeit "domtree 1" domtree = construct_domtree(ir.cfg.blocks) + @timeit "domtree 1" domtree = construct_domtree(ir) defuse_insts = scan_slot_def_use(nargs, ci, ir.stmts.stmt) 𝕃ₒ = optimizer_lattice(sv.inlining.interp) @timeit "construct_ssa" ir = construct_ssa!(ci, ir, sv, domtree, defuse_insts, 𝕃ₒ) # consumes `ir` diff --git a/base/compiler/ssair/ir.jl b/base/compiler/ssair/ir.jl index 80d6d5fde556d..ca658ad101f7f 100644 --- a/base/compiler/ssair/ir.jl +++ b/base/compiler/ssair/ir.jl @@ -397,6 +397,12 @@ function IRCode() return ir end +construct_domtree(ir::IRCode) = construct_domtree(ir.cfg) +construct_domtree(cfg::CFG) = construct_domtree(cfg.blocks) + +construct_postdomtree(ir::IRCode) = construct_postdomtree(ir.cfg) +construct_postdomtree(cfg::CFG) = construct_postdomtree(cfg.blocks) + function block_for_inst(ir::IRCode, inst::Int) if inst > length(ir.stmts) inst = ir.new_nodes.info[inst - length(ir.stmts)].pos diff --git a/test/compiler/irpasses.jl b/test/compiler/irpasses.jl index 9345c2e26db33..35aba9f647ea8 100644 --- a/test/compiler/irpasses.jl +++ b/test/compiler/irpasses.jl @@ -29,7 +29,7 @@ let code = Any[ ReturnNode(Core.SSAValue(10)), ] ir = make_ircode(code) - domtree = Core.Compiler.construct_domtree(ir.cfg.blocks) + domtree = Core.Compiler.construct_domtree(ir) ir = Core.Compiler.domsort_ssa!(ir, domtree) Core.Compiler.verify_ir(ir) phi = ir.stmts.stmt[3] @@ -47,7 +47,7 @@ let code = Any[] push!(code, Expr(:call, :opaque)) push!(code, ReturnNode(nothing)) ir = make_ircode(code) - domtree = Core.Compiler.construct_domtree(ir.cfg.blocks) + domtree = Core.Compiler.construct_domtree(ir) ir = Core.Compiler.domsort_ssa!(ir, domtree) Core.Compiler.verify_ir(ir) end diff --git a/test/compiler/ssair.jl b/test/compiler/ssair.jl index 0a53dec80f732..e237ab732ef57 100644 --- a/test/compiler/ssair.jl +++ b/test/compiler/ssair.jl @@ -73,8 +73,8 @@ let cfg = CFG(BasicBlock[ @test dfs.from_pre[dfs.to_parent_pre[dfs.to_pre[5]]] == 4 let correct_idoms = Compiler.naive_idoms(cfg.blocks), correct_pidoms = Compiler.naive_idoms(cfg.blocks, true) - @test Compiler.construct_domtree(cfg.blocks).idoms_bb == correct_idoms - @test Compiler.construct_postdomtree(cfg.blocks).idoms_bb == correct_pidoms + @test Compiler.construct_domtree(cfg).idoms_bb == correct_idoms + @test Compiler.construct_postdomtree(cfg).idoms_bb == correct_pidoms # For completeness, reverse the order of pred/succ in the CFG and verify # the answer doesn't change (it does change the which node is chosen # as the semi-dominator, since it changes the DFS numbering). @@ -85,8 +85,8 @@ let cfg = CFG(BasicBlock[ c && (blocks[4] = make_bb(reverse(blocks[4].preds), blocks[4].succs)) d && (blocks[5] = make_bb(reverse(blocks[5].preds), blocks[5].succs)) cfg′ = CFG(blocks, cfg.index) - @test Compiler.construct_domtree(cfg′.blocks).idoms_bb == correct_idoms - @test Compiler.construct_postdomtree(cfg′.blocks).idoms_bb == correct_pidoms + @test Compiler.construct_domtree(cfg′).idoms_bb == correct_idoms + @test Compiler.construct_postdomtree(cfg′).idoms_bb == correct_pidoms end end end @@ -292,7 +292,7 @@ let cfg = CFG(BasicBlock[ make_bb([2, 6], []), make_bb([4], [5, 3]), ], Int[]) - domtree = Compiler.construct_domtree(cfg.blocks) + domtree = Compiler.construct_domtree(cfg) @test domtree.dfs_tree.to_pre == [1, 2, 4, 5, 3, 6] @test domtree.idoms_bb == Compiler.naive_idoms(cfg.blocks) == [0, 1, 1, 3, 1, 4] @@ -486,7 +486,7 @@ let ir = Base.code_ircode((Bool,Any)) do c, x end end # domination analysis - domtree = Core.Compiler.construct_domtree(ir.cfg.blocks) + domtree = Core.Compiler.construct_domtree(ir) @test Core.Compiler.dominates(domtree, 1, 2) @test Core.Compiler.dominates(domtree, 1, 3) @test Core.Compiler.dominates(domtree, 1, 4) @@ -497,7 +497,7 @@ let ir = Base.code_ircode((Bool,Any)) do c, x end end # post domination analysis - post_domtree = Core.Compiler.construct_postdomtree(ir.cfg.blocks) + post_domtree = Core.Compiler.construct_postdomtree(ir) @test Core.Compiler.postdominates(post_domtree, 4, 1) @test Core.Compiler.postdominates(post_domtree, 4, 2) @test Core.Compiler.postdominates(post_domtree, 4, 3) diff --git a/test/compiler/tarjan.jl b/test/compiler/tarjan.jl index 44acafb2ca5f4..11c6b68e58b1b 100644 --- a/test/compiler/tarjan.jl +++ b/test/compiler/tarjan.jl @@ -102,7 +102,7 @@ function test_reachability(V, E; deletions = 2E ÷ 3, all_checks=false) end cfg = rand_cfg(V, E) - domtree = Core.Compiler.construct_domtree(cfg.blocks) + domtree = Core.Compiler.construct_domtree(cfg) reachability = CFGReachability(cfg, domtree) check_reachability(reachability, cfg, domtree, all_checks) From 334fd49eeb9f89ad59975a25fcd5846e833d925e Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Fri, 8 Mar 2024 16:25:36 +0100 Subject: [PATCH 27/55] bump Pkg --- .../Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/md5 | 1 + .../Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/sha512 | 1 + .../Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/md5 | 1 - .../Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/sha512 | 1 - stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/md5 create mode 100644 deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/sha512 diff --git a/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/md5 b/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/md5 new file mode 100644 index 0000000000000..32abf7f99b463 --- /dev/null +++ b/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/md5 @@ -0,0 +1 @@ +f9e5cafbfda75a03d700b165c03e5bb3 diff --git a/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/sha512 b/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/sha512 new file mode 100644 index 0000000000000..5fa17a3e0fdb5 --- /dev/null +++ b/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/sha512 @@ -0,0 +1 @@ +3353a9da04aae86b788209afd19ace7132d94fca4f777f6f13841630a1eb4c63fd9020e53a38b269c3d4f1d1548dafa848edc5b9081a34852afe1b3a0eb874ff diff --git a/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/md5 b/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/md5 deleted file mode 100644 index 3fc997ee6d795..0000000000000 --- a/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -de47c3414e7e3168f77b20676c3f7631 diff --git a/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/sha512 b/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/sha512 deleted file mode 100644 index 7d14fdc309cfe..0000000000000 --- a/deps/checksums/Pkg-56c379045c8cf876b365c284a5b442f2dc9fc8af.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -b4b7ebb7fe636bbb9d653cb826e1a74eadcbc936dbf1a76281a7cc2e439a17f48ea8d04b77d825deb94a0fe228490760df5836837cafbcbd7308251d0be6589f diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index a68d4a1697eac..64a456f678ad1 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 56c379045c8cf876b365c284a5b442f2dc9fc8af +PKG_SHA1 = 1e6fa605f5554f37d2ebf47de9763152f6f78232 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 06d5ca1d0c0f52b0d039e4ff2bdd9e0f442f175d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 8 Mar 2024 17:02:54 +0100 Subject: [PATCH 28/55] precompilepkgs: package in boths deps and weakdeps are in fact only weak (#53649) Missed when porting from Pkg. Fixes https://github.com/JuliaLang/Pkg.jl/issues/3834. (cherry picked from commit e618369409609f905d2a65f1e68cc934931991fd) --- base/precompilation.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/base/precompilation.jl b/base/precompilation.jl index 9eaa0539f522e..4994242481cff 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -26,6 +26,9 @@ struct ExplicitEnv end function ExplicitEnv(envpath::String=Base.active_project()) + if !isfile(envpath) + error("expected a project file at $(repr(envpath))") + end envpath = abspath(envpath) project_d = parsed_toml(envpath) @@ -51,6 +54,11 @@ function ExplicitEnv(envpath::String=Base.active_project()) end end + # A package in both deps and weakdeps is in fact only a weakdep + for (name, _) in project_weakdeps + delete!(project_deps, name) + end + project_extensions = Dict{String, Vector{UUID}}() # Collect all extensions of the project for (name, triggers::Union{String, Vector{String}}) in get(Dict{String, Any}, project_d, "extensions")::Dict{String, Any} From a66bbfa3eaa27bbb7c05eb3a0b048e3306be86eb Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Thu, 14 Mar 2024 01:23:57 +0900 Subject: [PATCH 29/55] inlining: remove ineffective handling for unmatched params (#52092) The deleted branch was added in #45062, although it had not been tested. I tried the following diff to find cases optimized by that, but I just found the handling proved to be in vain in all cases I tried. ```diff diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 318b21b09b..7e42a65aa4 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -1473,6 +1473,14 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig handle_any_const_result!(cases, result, match, argtypes, info, flag, state; allow_abstract=true, allow_typevars=true) fully_covered = handled_all_cases = match.fully_covers + if length(cases) == 1 && fully_covered + println("first case: ", only_method) + elseif length(cases) == 1 + atype = argtypes_to_type(sig.argtypes) + if atype isa DataType && cases[1].sig isa DataType + println("second case: ", only_method) + end + end elseif !handled_all_cases # if we've not seen all candidates, union split is valid only for dispatch tuples filter!(case::InliningCase->isdispatchtuple(case.sig), cases) ``` --- base/compiler/ssair/inlining.jl | 56 +++++++++------------------------ 1 file changed, 14 insertions(+), 42 deletions(-) diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 4d46c198d9964..f91062f31fb50 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -1352,12 +1352,10 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig nunion === nothing && return nothing cases = InliningCase[] argtypes = sig.argtypes - local handled_all_cases::Bool = true - local revisit_idx = local only_method = nothing - local meth::MethodLookupResult + local handled_all_cases = local fully_covered = true + local revisit_idx = nothing local all_result_count = 0 - local joint_effects::Effects = EFFECTS_TOTAL - local fully_covered::Bool = true + local joint_effects = EFFECTS_TOTAL for i = 1:nunion meth = getsplit(info, i) if meth.ambig @@ -1368,18 +1366,8 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig # No applicable methods; try next union split handled_all_cases = false continue - else - if length(meth) == 1 && only_method !== missing - if only_method === nothing - only_method = meth[1].method - elseif only_method !== meth[1].method - only_method = missing - end - else - only_method = missing - end end - local split_fully_covered::Bool = false + local split_fully_covered = false for (j, match) in enumerate(meth) all_result_count += 1 result = getresult(info, all_result_count) @@ -1406,33 +1394,17 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig (handled_all_cases & fully_covered) || (joint_effects = Effects(joint_effects; nothrow=false)) - if handled_all_cases && revisit_idx !== nothing - # we handled everything except one match with unmatched sparams, - # so try to handle it by bypassing validate_sparams - (i, j, k) = revisit_idx - match = getsplit(info, i)[j] - result = getresult(info, k) - handled_all_cases &= handle_any_const_result!(cases, - result, match, argtypes, info, flag, state; allow_abstract=true, allow_typevars=true) - elseif length(cases) == 0 && only_method isa Method - # if the signature is fully covered and there is only one applicable method, - # we can try to inline it even in the presence of unmatched sparams - # -- But don't try it if we already tried to handle the match in the revisit_idx - # case, because that'll (necessarily) be the same method. - if nsplit(info)::Int > 1 - atype = argtypes_to_type(argtypes) - (metharg, methsp) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), atype, only_method.sig)::SimpleVector - match = MethodMatch(metharg, methsp::SimpleVector, only_method, true) - result = nothing - else - @assert length(meth) == 1 - match = meth[1] - result = getresult(info, 1) + if handled_all_cases + if revisit_idx !== nothing + # we handled everything except one match with unmatched sparams, + # so try to handle it by bypassing validate_sparams + (i, j, k) = revisit_idx + match = getsplit(info, i)[j] + result = getresult(info, k) + handled_all_cases &= handle_any_const_result!(cases, + result, match, argtypes, info, flag, state; allow_abstract=true, allow_typevars=true) end - handle_any_const_result!(cases, - result, match, argtypes, info, flag, state; allow_abstract=true, allow_typevars=true) - fully_covered = handled_all_cases = match.fully_covers - elseif !handled_all_cases + elseif !isempty(cases) # if we've not seen all candidates, union split is valid only for dispatch tuples filter!(case::InliningCase->isdispatchtuple(case.sig), cases) end From 6e80df2b338f0bb74b330bcb4f47361fa34fca8e Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:39:56 +0900 Subject: [PATCH 30/55] inlining: bail out unless `match.spec_types <: match.method.sig` (#53720) As Jameson pointed out in the link below, while the union-split handles cases when there are uncovered matches, sometimes the expected condition `spec_types <: method.sig` that the union-split algorithm relies on isn't met in such cases, which caused issues like #52644. This commit fixes the problem by adding explicit checks for such cases. Note that this is based on #52092. The extra handling for single method match unmatched static parameters based on `only_method` that was removed in JuliaLang/#52092 has been ineffective and would otherwise cause problematic inlining on this PR. We'll likely need to come back to this later and figure out a safe and effective way to deal with such cases in the future when the handling for either case turns out to be necessary. - closes #52644 - xref: --- base/compiler/ssair/inlining.jl | 44 ++++++++++++++++++--------------- test/compiler/inline.jl | 21 ++++++++++++++++ 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index f91062f31fb50..a355e0b45c629 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -577,11 +577,11 @@ function ir_inline_unionsplit!(compact::IncrementalCompact, idx::Int, argexprs:: @assert nparams == fieldcount(mtype) if !(i == ncases && fully_covered) for i = 1:nparams - a, m = fieldtype(atype, i), fieldtype(mtype, i) + aft, mft = fieldtype(atype, i), fieldtype(mtype, i) # If this is always true, we don't need to check for it - a <: m && continue + aft <: mft && continue # Generate isa check - isa_expr = Expr(:call, isa, argexprs[i], m) + isa_expr = Expr(:call, isa, argexprs[i], mft) ssa = insert_node_here!(compact, NewInstruction(isa_expr, Bool, line)) if cond === true cond = ssa @@ -600,10 +600,10 @@ function ir_inline_unionsplit!(compact::IncrementalCompact, idx::Int, argexprs:: for i = 1:nparams argex = argexprs[i] (isa(argex, SSAValue) || isa(argex, Argument)) || continue - a, m = fieldtype(atype, i), fieldtype(mtype, i) - if !(a <: m) + aft, mft = fieldtype(atype, i), fieldtype(mtype, i) + if !(aft <: mft) argexprs′[i] = insert_node_here!(compact, - NewInstruction(PiNode(argex, m), m, line)) + NewInstruction(PiNode(argex, mft), mft, line)) end end end @@ -964,7 +964,8 @@ function analyze_method!(match::MethodMatch, argtypes::Vector{Any}, if !match.fully_covers # type-intersection was not able to give us a simple list of types, so # ir_inline_unionsplit won't be able to deal with inlining this - if !(spec_types isa DataType && length(spec_types.parameters) == length(argtypes) && !isvarargtype(spec_types.parameters[end])) + if !(spec_types isa DataType && length(spec_types.parameters) == npassedargs && + !isvarargtype(spec_types.parameters[end])) return nothing end end @@ -1374,16 +1375,18 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig joint_effects = merge_effects(joint_effects, info_effects(result, match, state)) split_fully_covered |= match.fully_covers if !validate_sparams(match.sparams) - if !match.fully_covers - handled_all_cases = false - continue - end - if revisit_idx === nothing - revisit_idx = (i, j, all_result_count) + if match.fully_covers + if revisit_idx === nothing + revisit_idx = (i, j, all_result_count) + else + handled_all_cases = false + revisit_idx = nothing + end else handled_all_cases = false - revisit_idx = nothing end + elseif !(match.spec_types <: match.method.sig) # the requirement for correct union-split + handled_all_cases = false else handled_all_cases &= handle_any_const_result!(cases, result, match, argtypes, info, flag, state; allow_abstract=true, allow_typevars=false) @@ -1418,14 +1421,15 @@ function handle_call!(todo::Vector{Pair{Int,Any}}, cases = compute_inlining_cases(info, flag, sig, state) cases === nothing && return nothing cases, all_covered, joint_effects = cases - handle_cases!(todo, ir, idx, stmt, argtypes_to_type(sig.argtypes), cases, - all_covered, joint_effects) + atype = argtypes_to_type(sig.argtypes) + handle_cases!(todo, ir, idx, stmt, atype, cases, all_covered, joint_effects) end function handle_match!(cases::Vector{InliningCase}, match::MethodMatch, argtypes::Vector{Any}, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; - allow_abstract::Bool, allow_typevars::Bool, volatile_inf_result::Union{Nothing,VolatileInferenceResult}) + allow_abstract::Bool, allow_typevars::Bool, + volatile_inf_result::Union{Nothing,VolatileInferenceResult}) spec_types = match.spec_types allow_abstract || isdispatchtuple(spec_types) || return false # We may see duplicated dispatch signatures here when a signature gets widened @@ -1512,19 +1516,19 @@ function concrete_result_item(result::ConcreteResult, @nospecialize(info::CallIn end function handle_cases!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, - @nospecialize(atype), cases::Vector{InliningCase}, fully_covered::Bool, + @nospecialize(atype), cases::Vector{InliningCase}, all_covered::Bool, joint_effects::Effects) # If we only have one case and that case is fully covered, we may either # be able to do the inlining now (for constant cases), or push it directly # onto the todo list - if fully_covered && length(cases) == 1 + if all_covered && length(cases) == 1 handle_single_case!(todo, ir, idx, stmt, cases[1].item) elseif length(cases) > 0 isa(atype, DataType) || return nothing for case in cases isa(case.sig, DataType) || return nothing end - push!(todo, idx=>UnionSplit(fully_covered, atype, cases)) + push!(todo, idx=>UnionSplit(all_covered, atype, cases)) else add_flag!(ir[SSAValue(idx)], flags_for_effects(joint_effects)) end diff --git a/test/compiler/inline.jl b/test/compiler/inline.jl index d4c6659593fe6..cca1072939025 100644 --- a/test/compiler/inline.jl +++ b/test/compiler/inline.jl @@ -2159,3 +2159,24 @@ end @test !Core.Compiler.is_nothrow(Base.infer_effects(issue53062, (Bool,))) @test issue53062(false) == -1 @test_throws MethodError issue53062(true) + +struct Issue52644 + tuple::Type{<:Tuple} +end +issue52644(::DataType) = :DataType +issue52644(::UnionAll) = :UnionAll +let ir = Base.code_ircode((Issue52644,); optimize_until="Inlining") do t + issue52644(t.tuple) + end |> only |> first + irfunc = Core.OpaqueClosure(ir) + @test irfunc(Issue52644(Tuple{})) === :DataType + @test irfunc(Issue52644(Tuple{<:Integer})) === :UnionAll +end +issue52644_single(x::DataType) = :DataType +let ir = Base.code_ircode((Issue52644,); optimize_until="Inlining") do t + issue52644_single(t.tuple) + end |> only |> first + irfunc = Core.OpaqueClosure(ir) + @test irfunc(Issue52644(Tuple{})) === :DataType + @test_throws MethodError irfunc(Issue52644(Tuple{<:Integer})) +end From 5adc0f6d94442fc6f79f82cc8e6358d0f99587d3 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Sun, 18 Feb 2024 14:10:51 +0530 Subject: [PATCH 31/55] Orthogonalize re-indexing for FastSubArrays (#53369) By separating out the re-indexing step for `FastSubArray`s and specializing this for `FastContiguousSubArray`s, we don't need to define specialized `getindex`, `setindex!` and `isassigned` for `FastContiguousSubArray`s anymore. The fallback method for `FastSubArray`s will correctly handle the special case. (cherry picked from commit 16871e73f813345b2790c2879ececde55a4a2d65) --- base/subarray.jl | 86 ++++++++++++++++-------------------------------- 1 file changed, 28 insertions(+), 58 deletions(-) diff --git a/base/subarray.jl b/base/subarray.jl index eca06fa3eacff..396f7c52bd77a 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -320,22 +320,35 @@ end # But SubArrays with fast linear indexing pre-compute a stride and offset FastSubArray{T,N,P,I} = SubArray{T,N,P,I,true} +# We define a convenience functions to compute the shifted parent index +# This differs from reindex as this accepts the view directly, instead of its indices +@inline _reindexlinear(V::FastSubArray, i::Int) = V.offset1 + V.stride1*i +@inline _reindexlinear(V::FastSubArray, i::AbstractUnitRange{Int}) = V.offset1 .+ V.stride1 .* i + function getindex(V::FastSubArray, i::Int) @inline @boundscheck checkbounds(V, i) - @inbounds r = V.parent[V.offset1 + V.stride1*i] + @inbounds r = V.parent[_reindexlinear(V, i)] r end -# We can avoid a multiplication if the first parent index is a Colon or AbstractUnitRange, -# or if all the indices are scalars, i.e. the view is for a single value only -FastContiguousSubArray{T,N,P,I<:Union{Tuple{Union{Slice, AbstractUnitRange}, Vararg{Any}}, - Tuple{Vararg{ScalarIndex}}}} = SubArray{T,N,P,I,true} -function getindex(V::FastContiguousSubArray, i::Int) + +# For vector views with linear indexing, we disambiguate to favor the stride/offset +# computation as that'll generally be faster than (or just as fast as) re-indexing into a range. +function getindex(V::FastSubArray{<:Any, 1}, i::Int) @inline @boundscheck checkbounds(V, i) - @inbounds r = V.parent[V.offset1 + i] + @inbounds r = V.parent[_reindexlinear(V, i)] r end + +# We can avoid a multiplication if the first parent index is a Colon or AbstractUnitRange, +# or if all the indices are scalars, i.e. the view is for a single value only +FastContiguousSubArray{T,N,P,I<:Union{Tuple{Union{Slice, AbstractUnitRange}, Vararg{Any}}, + Tuple{Vararg{ScalarIndex}}}} = SubArray{T,N,P,I,true} + +@inline _reindexlinear(V::FastContiguousSubArray, i::Int) = V.offset1 + i +@inline _reindexlinear(V::FastContiguousSubArray, i::AbstractUnitRange{Int}) = V.offset1 .+ i + # parents of FastContiguousSubArrays may support fast indexing with AbstractUnitRanges, # so we may just forward the indexing to the parent # This may only be done for non-offset ranges, as the result would otherwise have offset axes @@ -343,24 +356,10 @@ const OneBasedRanges = Union{OneTo{Int}, UnitRange{Int}, Slice{OneTo{Int}}, Iden function getindex(V::FastContiguousSubArray, i::OneBasedRanges) @inline @boundscheck checkbounds(V, i) - @inbounds r = V.parent[V.offset1 .+ i] + @inbounds r = V.parent[_reindexlinear(V, i)] r end -# For vector views with linear indexing, we disambiguate to favor the stride/offset -# computation as that'll generally be faster than (or just as fast as) re-indexing into a range. -function getindex(V::FastSubArray{<:Any, 1}, i::Int) - @inline - @boundscheck checkbounds(V, i) - @inbounds r = V.parent[V.offset1 + V.stride1*i] - r -end -function getindex(V::FastContiguousSubArray{<:Any, 1}, i::Int) - @inline - @boundscheck checkbounds(V, i) - @inbounds r = V.parent[V.offset1 + i] - r -end @inline getindex(V::FastContiguousSubArray, i::Colon) = getindex(V, to_indices(V, (:,))...) # Indexed assignment follows the same pattern as `getindex` above @@ -373,40 +372,23 @@ end function setindex!(V::FastSubArray, x, i::Int) @inline @boundscheck checkbounds(V, i) - @inbounds V.parent[V.offset1 + V.stride1*i] = x + @inbounds V.parent[_reindexlinear(V, i)] = x V end -function setindex!(V::FastContiguousSubArray, x, i::Int) +function setindex!(V::FastSubArray{<:Any, 1}, x, i::Int) @inline @boundscheck checkbounds(V, i) - @inbounds V.parent[V.offset1 + i] = x + @inbounds V.parent[_reindexlinear(V, i)] = x V end + function setindex!(V::FastSubArray, x, i::AbstractUnitRange{Int}) @inline @boundscheck checkbounds(V, i) - @inbounds V.parent[V.offset1 .+ V.stride1 .* i] = x - V -end -function setindex!(V::FastContiguousSubArray, x, i::AbstractUnitRange{Int}) - @inline - @boundscheck checkbounds(V, i) - @inbounds V.parent[V.offset1 .+ i] = x + @inbounds V.parent[_reindexlinear(V, i)] = x V end -function setindex!(V::FastSubArray{<:Any, 1}, x, i::Int) - @inline - @boundscheck checkbounds(V, i) - @inbounds V.parent[V.offset1 + V.stride1*i] = x - V -end -function setindex!(V::FastContiguousSubArray{<:Any, 1}, x, i::Int) - @inline - @boundscheck checkbounds(V, i) - @inbounds V.parent[V.offset1 + i] = x - V -end @inline setindex!(V::FastSubArray, x, i::Colon) = setindex!(V, x, to_indices(V, (i,))...) function isassigned(V::SubArray{T,N}, I::Vararg{Int,N}) where {T,N} @@ -418,25 +400,13 @@ end function isassigned(V::FastSubArray, i::Int) @inline @boundscheck checkbounds(Bool, V, i) || return false - @inbounds r = isassigned(V.parent, V.offset1 + V.stride1*i) - r -end -function isassigned(V::FastContiguousSubArray, i::Int) - @inline - @boundscheck checkbounds(Bool, V, i) || return false - @inbounds r = isassigned(V.parent, V.offset1 + i) + @inbounds r = isassigned(V.parent, _reindexlinear(V, i)) r end function isassigned(V::FastSubArray{<:Any, 1}, i::Int) @inline @boundscheck checkbounds(Bool, V, i) || return false - @inbounds r = isassigned(V.parent, V.offset1 + V.stride1*i) - r -end -function isassigned(V::FastContiguousSubArray{<:Any, 1}, i::Int) - @inline - @boundscheck checkbounds(Bool, V, i) || return false - @inbounds r = isassigned(V.parent, V.offset1 + i) + @inbounds r = isassigned(V.parent, _reindexlinear(V, i)) r end From 12add8d61f5e1650ef5dc2e20779b1ba1f8680b4 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Thu, 7 Mar 2024 09:07:59 -0600 Subject: [PATCH 32/55] typo fix in scoped values docs (#53629) (cherry picked from commit cf5f1632933a146c82ae9b33e76d89241d976dbd) --- doc/src/base/scopedvalues.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index 2991522900e1a..8c3340ab9cf11 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -295,6 +295,6 @@ version of Julia. ## Design inspiration This design was heavily inspired by [JEPS-429](https://openjdk.org/jeps/429), -which in turn was inspired by dynamically scoped free variables in many Lisp dialects. In particular Interlisp-D and it's deep binding strategy. +which in turn was inspired by dynamically scoped free variables in many Lisp dialects. In particular Interlisp-D and its deep binding strategy. A prior design discussed was context variables ala [PEPS-567](https://peps.python.org/pep-0567/) and implemented in Julia as [ContextVariablesX.jl](https://github.com/tkf/ContextVariablesX.jl). From 90454b2cb9335378805cbd9b80915d8a4657a277 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 8 Mar 2024 01:26:10 -0500 Subject: [PATCH 33/55] sroa: Fix incorrect scope counting (#53630) Sroa was incorrectly assuming that every :leave leaves exactly one scope. In reality, it leaves as many scopes as the corresponding :leave references. Fix that to fix #53521. (cherry picked from commit 321fb2ccda65aa122c3549ef4951c6890b440cf2) --- base/compiler/ssair/passes.jl | 17 ++++++++++++++--- test/compiler/irpasses.jl | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 5ea21c85427be..6f20ca6b20ec8 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1225,10 +1225,21 @@ function sroa_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) bb = compact.active_result_bb - 1 bbs = scope_mapping[bb] if isexpr(stmt, :leave) && bbs != SSAValue(0) - update_scope_mapping!(scope_mapping, bb+1, scope_mapping[block_for_inst(compact, bbs)]) - else - update_scope_mapping!(scope_mapping, bb+1, bbs) + # Here we want to count the number of scopes that we're leaving, + # which is the same as the number of EnterNodes being referenced + # by `stmt.args`. Which have :scope set. In practice, the frontend + # does emit these in order, so we could simply go to the last one, + # but we want to avoid making that semantic assumption. + for i = 1:length(stmt.args) + scope = stmt.args[i] + scope === nothing && continue + enter = compact[scope][:inst] + @assert isa(enter, EnterNode) + isdefined(enter, :scope) || continue + bbs = scope_mapping[block_for_inst(compact, bbs)] + end end + update_scope_mapping!(scope_mapping, bb+1, bbs) end # check whether this statement is `getfield` / `setfield!` (or other "interesting" statement) is_setfield = is_isdefined = is_finalizer = is_keyvalue_get = false diff --git a/test/compiler/irpasses.jl b/test/compiler/irpasses.jl index 35aba9f647ea8..c124832030cbd 100644 --- a/test/compiler/irpasses.jl +++ b/test/compiler/irpasses.jl @@ -1801,3 +1801,22 @@ let n = 1000 end end @test f_1000_blocks() == 0 + +# https://github.com/JuliaLang/julia/issues/53521 +# Incorrect scope counting in :leave +using Base.ScopedValues +function f53521() + VALUE = ScopedValue(1) + @with VALUE => 2 begin + for i = 1 + @with VALUE => 3 begin + try + foo() + catch + nothing + end + end + end + end +end +@test code_typed(f53521)[1][2] === Nothing From b147539a7910d1ef1f7b10568c8f2ab28a66149d Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 8 Mar 2024 10:21:33 -0500 Subject: [PATCH 34/55] Use Base parallel precompilation to build stdlibs (#53598) Follow-on from https://github.com/JuliaLang/julia/pull/53403 This extends `Base.Precompilation.precompilepkgs` to take a list of configurations to precompile each package with, while parallelizing across all packages and configurations, and uses it to build the stdlib pkgimages. It simplifies the stdlib pkgimage build process but is (currently) dependent on having an accurately resolved Manifest.toml (Project.toml included to make the manifest easier to make). Any new/removed stdlibs or changes their dependencies will require updating the Manifest.toml. It's a bit chicken and egg, but should be manageable with manual editing of the Manifest.toml. In terms of speed improvement: MacOS aarch64 CI runner 6m19s before, 5m19 with this Note that CI builds will show the basic print with timing of each package, whereas local build will be the tidier fancy print without timings. Co-authored-by: Valentin Churavy (cherry picked from commit 78351b5c478b65295dfa143d33ddc41f546baceb) --- Makefile | 3 +- base/loading.jl | 33 ++++- base/precompilation.jl | 318 +++++++++++++++++++++++------------------ base/util.jl | 1 + pkgimage.mk | 141 ++---------------- stdlib/Manifest.toml | 300 ++++++++++++++++++++++++++++++++++++++ stdlib/Project.toml | 61 ++++++++ stdlib/stdlib.mk | 21 ++- sysimage.mk | 2 +- 9 files changed, 589 insertions(+), 291 deletions(-) create mode 100644 stdlib/Manifest.toml create mode 100644 stdlib/Project.toml diff --git a/Makefile b/Makefile index 5ce3003dba231..d7d25998688c5 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink j julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest julia-base-cache stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-% - @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f pkgimage.mk all-$* + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f pkgimage.mk $* debug release : % : julia-% stdlibs-cache-% @@ -588,6 +588,7 @@ clean: | $(CLEAN_TARGETS) @-$(MAKE) -C $(BUILDROOT)/cli clean @-$(MAKE) -C $(BUILDROOT)/test clean @-$(MAKE) -C $(BUILDROOT)/stdlib clean + @-$(MAKE) -C $(BUILDROOT) -f pkgimage.mk clean -rm -f $(BUILDROOT)/julia -rm -f $(BUILDROOT)/*.tar.gz -rm -f $(build_depsbindir)/stringreplace \ diff --git a/base/loading.jl b/base/loading.jl index 5f3c846131810..c3c2647adbe4e 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1524,7 +1524,21 @@ function CacheFlags(f::UInt8) CacheFlags(use_pkgimages, debug_level, check_bounds, inline, opt_level) end CacheFlags(f::Int) = CacheFlags(UInt8(f)) -CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ())) +function CacheFlags(cf::CacheFlags=CacheFlags(ccall(:jl_cache_flags, UInt8, ())); + use_pkgimages::Union{Nothing,Bool}=nothing, + debug_level::Union{Nothing,Int}=nothing, + check_bounds::Union{Nothing,Int}=nothing, + inline::Union{Nothing,Bool}=nothing, + opt_level::Union{Nothing,Int}=nothing + ) + return CacheFlags( + use_pkgimages === nothing ? cf.use_pkgimages : use_pkgimages, + debug_level === nothing ? cf.debug_level : debug_level, + check_bounds === nothing ? cf.check_bounds : check_bounds, + inline === nothing ? cf.inline : inline, + opt_level === nothing ? cf.opt_level : opt_level + ) +end function _cacheflag_to_uint8(cf::CacheFlags)::UInt8 f = UInt8(0) @@ -2768,7 +2782,7 @@ function compilecache_dir(pkg::PkgId) return joinpath(DEPOT_PATH[1], entrypath) end -function compilecache_path(pkg::PkgId, prefs_hash::UInt64; project::String=something(Base.active_project(), ""))::String +function compilecache_path(pkg::PkgId, prefs_hash::UInt64; flags::CacheFlags=CacheFlags(), project::String=something(Base.active_project(), ""))::String entrypath, entryfile = cache_file_entry(pkg) cachepath = joinpath(DEPOT_PATH[1], entrypath) isdir(cachepath) || mkpath(cachepath) @@ -2778,7 +2792,7 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64; project::String=somet crc = _crc32c(project) crc = _crc32c(unsafe_string(JLOptions().image_file), crc) crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc) - crc = _crc32c(ccall(:jl_cache_flags, UInt8, ()), crc) + crc = _crc32c(_cacheflag_to_uint8(flags), crc) cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) if cpu_target === nothing @@ -2810,7 +2824,8 @@ end const MAX_NUM_PRECOMPILE_FILES = Ref(10) function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, internal_stdout::IO = stdout, - keep_loaded_modules::Bool = true; flags::Cmd=``, reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}()) + keep_loaded_modules::Bool = true; flags::Cmd=``, cacheflags::CacheFlags=CacheFlags(), + reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}()) @nospecialize internal_stderr internal_stdout # decide where to put the resulting cache file @@ -2859,7 +2874,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in # Read preferences hash back from .ji file (we can't precompute because # we don't actually know what the list of compile-time preferences are without compiling) prefs_hash = preferences_hash(tmppath) - cachefile = compilecache_path(pkg, prefs_hash) + cachefile = compilecache_path(pkg, prefs_hash; flags=cacheflags) ocachefile = cache_objects ? ocachefile_from_cachefile(cachefile) : nothing # append checksum for so to the end of the .ji file: @@ -3403,7 +3418,7 @@ global parse_pidfile_hook # The preferences hash is only known after precompilation so just assume no preferences. # Also ignore the active project, which means that if all other conditions are equal, # the same package cannot be precompiled from different projects and/or different preferences at the same time. -compilecache_pidfile_path(pkg::PkgId) = compilecache_path(pkg, UInt64(0); project="") * ".pidfile" +compilecache_pidfile_path(pkg::PkgId; flags::CacheFlags=CacheFlags()) = compilecache_path(pkg, UInt64(0); project="", flags) * ".pidfile" const compilecache_pidlock_stale_age = 10 @@ -3531,12 +3546,16 @@ end M = root_module(req_key) if PkgId(M) == req_key && module_build_id(M) === req_build_id depmods[i] = M + elseif M == Core + @debug "Rejecting cache file $cachefile because it was made with a different julia version" + record_reason(reasons, "wrong julia version") + return true # Won't be able to fulfill dependency elseif ignore_loaded || !stalecheck # Used by Pkg.precompile given that there it's ok to precompile different versions of loaded packages @goto locate_branch else @debug "Rejecting cache file $cachefile because module $req_key is already loaded and incompatible." - record_reason(reasons, req_key == PkgId(Core) ? "wrong julia version" : "wrong dep version loaded") + record_reason(reasons, "wrong dep version loaded") return true # Won't be able to fulfill dependency end else diff --git a/base/precompilation.jl b/base/precompilation.jl index 4994242481cff..9ecb190a1afb5 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -343,16 +343,29 @@ function printpkgstyle(io, header, msg; color=:light_green) println(io, " ", msg) end +const Config = Pair{Cmd, Base.CacheFlags} +const PkgConfig = Tuple{Base.PkgId,Config} + +function precompilepkgs(pkgs::Vector{String}=String[]; + internal_call::Bool=false, + strict::Bool = false, + warn_loaded::Bool = true, + timing::Bool = false, + _from_loading::Bool=false, + configs::Union{Config,Vector{Config}}=(``=>Base.CacheFlags()), + io::IO=stderr, + flags_cacheflags=nothing) + + if flags_cacheflags !== nothing + # TODO: Remove `flags_cacheflags` once Pkg moves to the renamed kwarg `configs` + configs = flags_cacheflags + end + configs = configs isa Config ? [configs] : configs -function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false, - strict::Bool=false, warn_loaded = true, timing::Bool = false, - _from_loading::Bool=false, flags_cacheflags::Pair{Cmd, Base.CacheFlags}=(``=>Base.CacheFlags()), io::IO=stderr) time_start = time_ns() env = ExplicitEnv() - flags, cacheflags = flags_cacheflags - # Windows sometimes hits a ReadOnlyMemoryError, so we halve the default number of tasks. Issue #2323 # TODO: Investigate why this happens in windows and restore the full task limit default_num_tasks = Sys.iswindows() ? div(Sys.CPU_THREADS::Int, 2) + 1 : Sys.CPU_THREADS::Int + 1 @@ -364,7 +377,8 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false # asking for timing disables fancy mode, as timing is shown in non-fancy mode fancyprint = can_fancyprint(io) && !timing - if _from_loading && !Sys.isinteractive() + if _from_loading && !Sys.isinteractive() && Base.get_bool_env("JULIA_TESTS", false) + # suppress passive loading printing in julia test suite. `JULIA_TESTS` is set in Base.runtests io = devnull end @@ -449,13 +463,16 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false end # initialize signalling - started = Dict{Base.PkgId,Bool}() - was_processed = Dict{Base.PkgId,Base.Event}() - was_recompiled = Dict{Base.PkgId,Bool}() - for pkgid in keys(depsmap) - started[pkgid] = false - was_processed[pkgid] = Base.Event() - was_recompiled[pkgid] = false + started = Dict{PkgConfig,Bool}() + was_processed = Dict{PkgConfig,Base.Event}() + was_recompiled = Dict{PkgConfig,Bool}() + for config in configs + for pkgid in keys(depsmap) + dep_config = (pkgid, config) + started[dep_config] = false + was_processed[dep_config] = Base.Event() + was_recompiled[dep_config] = false + end end @debug "precompile: signalling initialized" _group=:precompile @@ -489,7 +506,10 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false for pkg in keys(depsmap) if scan_pkg!(pkg, depsmap) push!(circular_deps, pkg) - notify(was_processed[pkg]) + for dep_config in keys(was_processed) + # notify all to allow skipping + dep_config[1] == pkg && notify(was_processed[dep_config]) + end end end if !isempty(circular_deps) @@ -534,14 +554,18 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false end target = join(pkgs, ", ") else - target = "project..." + target = "project" + end + if length(configs) > 1 || !isempty(only(configs)[1]) # if multiple configs or only one is not default + target *= " for $(length(configs)) compilation configurations..." + else + target *= "..." end @debug "precompile: packages filtered" _group=:precompile - pkg_queue = Base.PkgId[] - failed_deps = Dict{Base.PkgId, String}() - skipped_deps = Base.PkgId[] - precomperr_deps = Base.PkgId[] # packages that may succeed after a restart (i.e. loaded packages with no cache file) + pkg_queue = PkgConfig[] + failed_deps = Dict{PkgConfig, String}() + precomperr_deps = PkgConfig[] # packages that may succeed after a restart (i.e. loaded packages with no cache file) print_lock = io isa Base.LibuvStream ? io.lock::ReentrantLock : ReentrantLock() first_started = Base.Event() @@ -564,7 +588,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false in_printloop || wait(t_print) # wait to let the print loop cease first if err isa InterruptException lock(print_lock) do - println(io, " Interrupted: Exiting precompilation...") + println(io, " Interrupted: Exiting precompilation...", ansi_cleartoendofline) end interrupted = true return true @@ -572,13 +596,13 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false return false end end - - std_outputs = Dict{Base.PkgId,String}() - taskwaiting = Set{Base.PkgId}() - pkgspidlocked = Dict{Base.PkgId,String}() + std_outputs = Dict{PkgConfig,String}() + taskwaiting = Set{PkgConfig}() + pkgspidlocked = Dict{PkgConfig,String}() pkg_liveprinted = nothing - function monitor_std(pkg, pipe; single_requested_pkg=false) + function monitor_std(pkg_config, pipe; single_requested_pkg=false) + pkg, config = pkg_config try liveprinting = false while !eof(pipe) @@ -594,14 +618,14 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false print(io, ansi_cleartoendofline, str) end end - std_outputs[pkg] = string(get(std_outputs, pkg, ""), str) - if !in(pkg, taskwaiting) && occursin("waiting for IO to finish", str) + std_outputs[pkg_config] = string(get(std_outputs, pkg_config, ""), str) + if !in(pkg_config, taskwaiting) && occursin("waiting for IO to finish", str) !fancyprint && lock(print_lock) do println(io, pkg.name, color_string(" Waiting for background task / IO / timer.", Base.warn_color())) end - push!(taskwaiting, pkg) + push!(taskwaiting, pkg_config) end - if !fancyprint && in(pkg, taskwaiting) + if !fancyprint && in(pkg_config, taskwaiting) lock(print_lock) do print(io, str) end @@ -626,7 +650,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false i = 1 last_length = 0 bar = MiniProgressBar(; indent=2, header = "Progress", color = Base.info_color(), percentage=false, always_reprint=true) - n_total = length(depsmap) + n_total = length(depsmap) * length(configs) bar.max = n_total - n_already_precomp final_loop = false n_print_rows = 0 @@ -652,30 +676,35 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false str = sprint(io -> show_progress(io, bar; termwidth, carriagereturn=false); context=io) print(iostr, Base._truncate_at_width_or_chars(true, str, termwidth), "\n") end - for dep in pkg_queue_show + for dep_config in pkg_queue_show + dep, config = dep_config loaded = warn_loaded && haskey(Base.loaded_modules, dep) _name = haskey(exts, dep) ? string(exts[dep], " → ", dep.name) : dep.name name = dep in direct_deps ? _name : string(color_string(_name, :light_black)) - line = if dep in precomperr_deps + if length(configs) > 1 + config_str = isempty(config[1]) ? "" : "$(join(config[1], " "))" + name *= color_string(" $(config_str)", :light_black) + end + line = if dep_config in precomperr_deps string(color_string(" ? ", Base.warn_color()), name) - elseif haskey(failed_deps, dep) + elseif haskey(failed_deps, dep_config) string(color_string(" ✗ ", Base.error_color()), name) - elseif was_recompiled[dep] + elseif was_recompiled[dep_config] !loaded && interrupted_or_done.set && continue loaded || @async begin # keep successful deps visible for short period sleep(1); - filter!(!isequal(dep), pkg_queue) + filter!(!isequal(dep_config), pkg_queue) end string(color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) - elseif started[dep] + elseif started[dep_config] # Offset each spinner animation using the first character in the package name as the seed. # If not offset, on larger terminal fonts it looks odd that they all sync-up anim_char = anim_chars[(i + Int(dep.name[1])) % length(anim_chars) + 1] anim_char_colored = dep in direct_deps ? anim_char : color_string(anim_char, :light_black) - waiting = if haskey(pkgspidlocked, dep) - who_has_lock = pkgspidlocked[dep] + waiting = if haskey(pkgspidlocked, dep_config) + who_has_lock = pkgspidlocked[dep_config] color_string(" Being precompiled by $(who_has_lock)", Base.info_color()) - elseif dep in taskwaiting + elseif dep_config in taskwaiting color_string(" Waiting for background task / IO / timer. Interrupt to inspect", Base.warn_color()) else "" @@ -712,103 +741,108 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false for (pkg, deps) in depsmap cachepaths = Base.find_all_in_cache_path(pkg) sourcepath = Base.locate_package(pkg) - if sourcepath === nothing - failed_deps[pkg] = "Error: Missing source file for $(pkg)" - notify(was_processed[pkg]) - continue - end - # Heuristic for when precompilation is disabled - if occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcepath, String)) - notify(was_processed[pkg]) - continue - end - single_requested_pkg = length(pkgs) == 1 && only(pkgs) == pkg.name - - task = @async begin - try - loaded = haskey(Base.loaded_modules, pkg) - for dep in deps # wait for deps to finish - wait(was_processed[dep]) - end - - circular = pkg in circular_deps - is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepaths, sourcepath, flags=cacheflags) - if !circular && is_stale - Base.acquire(parallel_limiter) - is_direct_dep = pkg in direct_deps - - # std monitoring - std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) - t_monitor = @async monitor_std(pkg, std_pipe; single_requested_pkg) - - _name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name - name = is_direct_dep ? _name : string(color_string(_name, :light_black)) - !fancyprint && lock(print_lock) do - isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) - end - push!(pkg_queue, pkg) - started[pkg] = true - fancyprint && notify(first_started) - if interrupted_or_done.set - notify(was_processed[pkg]) - Base.release(parallel_limiter) - return + for config in configs + pkg_config = (pkg, config) + if sourcepath === nothing + failed_deps[pkg_config] = "Error: Missing source file for $(pkg)" + notify(was_processed[pkg_config]) + continue + end + # Heuristic for when precompilation is disabled + if occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcepath, String)) + notify(was_processed[pkg_config]) + continue + end + flags, cacheflags = config + task = @async begin + try + loaded = haskey(Base.loaded_modules, pkg) + for dep in deps # wait for deps to finish + wait(was_processed[(dep,config)]) end - try - # allows processes to wait if another process is precompiling a given package to - # a functionally identical package cache (except for preferences, which may differ) - t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg, pkgspidlocked, hascolor, cacheflags) do - Base.with_logger(Base.NullLogger()) do - # The false here means we ignore loaded modules, so precompile for a fresh session - Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags) - end + circular = pkg in circular_deps + is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepaths, sourcepath, flags=cacheflags) + if !circular && is_stale + Base.acquire(parallel_limiter) + is_direct_dep = pkg in direct_deps + + # std monitoring + std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) + t_monitor = @async monitor_std(pkg_config, std_pipe; single_requested_pkg) + + _name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name + name = is_direct_dep ? _name : string(color_string(_name, :light_black)) + if length(configs) > 1 + config_str = isempty(config[1]) ? "" : "$(join(config[1], " "))" + name *= color_string(" $(config_str)", :light_black) + end + !fancyprint && lock(print_lock) do + isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) end - if ret isa Base.PrecompilableError - push!(precomperr_deps, pkg) - !fancyprint && lock(print_lock) do - println(io, _timing_string(t), color_string(" ? ", Base.warn_color()), name) + push!(pkg_queue, pkg_config) + started[pkg_config] = true + fancyprint && notify(first_started) + if interrupted_or_done.set + notify(was_processed[pkg_config]) + Base.release(parallel_limiter) + return + end + try + # allows processes to wait if another process is precompiling a given package to + # a functionally identical package cache (except for preferences, which may differ) + t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg_config, pkgspidlocked, hascolor) do + Base.with_logger(Base.NullLogger()) do + # The false here means we ignore loaded modules, so precompile for a fresh session + Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags, cacheflags) + end end - else - !fancyprint && lock(print_lock) do - println(io, _timing_string(t), color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) + if ret isa Base.PrecompilableError + push!(precomperr_deps, pkg_config) + !fancyprint && lock(print_lock) do + println(io, _timing_string(t), color_string(" ? ", Base.warn_color()), name) + end + else + !fancyprint && lock(print_lock) do + println(io, _timing_string(t), color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) + end + was_recompiled[pkg_config] = true end - was_recompiled[pkg] = true - end - loaded && (n_loaded += 1) - catch err - close(std_pipe.in) # close pipe to end the std output monitor - wait(t_monitor) - if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file")) - failed_deps[pkg] = (strict || is_direct_dep) ? string(sprint(showerror, err), "\n", strip(get(std_outputs, pkg, ""))) : "" - delete!(std_outputs, pkg) # so it's not shown as warnings, given error report - !fancyprint && lock(print_lock) do - println(io, " "^9, color_string(" ✗ ", Base.error_color()), name) + loaded && (n_loaded += 1) + catch err + close(std_pipe.in) # close pipe to end the std output monitor + wait(t_monitor) + if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file")) + failed_deps[dep_config] = (strict || is_direct_dep) ? string(sprint(showerror, err), "\n", strip(get(std_outputs, pkg, ""))) : "" + delete!(std_outputs, pkg_config) # so it's not shown as warnings, given error report + !fancyprint && lock(print_lock) do + println(io, " "^9, color_string(" ✗ ", Base.error_color()), name) + end + else + rethrow() end - else - rethrow() + finally + isopen(std_pipe.in) && close(std_pipe.in) # close pipe to end the std output monitor + wait(t_monitor) + Base.release(parallel_limiter) end - finally - isopen(std_pipe.in) && close(std_pipe.in) # close pipe to end the std output monitor - wait(t_monitor) - Base.release(parallel_limiter) + else + is_stale || (n_already_precomp += 1) end - else - is_stale || (n_already_precomp += 1) + n_done += 1 + notify(was_processed[pkg_config]) + catch err_outer + # For debugging: + # println("Task failed $err_outer") # logging doesn't show here + handle_interrupt(err_outer) || rethrow() + notify(was_processed[pkg_config]) + finally + filter!(!istaskdone, tasks) + length(tasks) == 1 && notify(interrupted_or_done) end - n_done += 1 - notify(was_processed[pkg]) - catch err_outer - # For debugging: - # @error "Task failed" exception=(err_outer, catch_backtrace()) - handle_interrupt(err_outer) || rethrow() - notify(was_processed[pkg]) - finally - filter!(!istaskdone, tasks) - length(tasks) == 1 && notify(interrupted_or_done) end + push!(tasks, task) end - push!(tasks, task) end isempty(tasks) && notify(interrupted_or_done) try @@ -826,28 +860,27 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false if ndeps > 0 || !isempty(failed_deps) || (quick_exit && !isempty(std_outputs)) str = sprint() do iostr if !quick_exit - plural = ndeps == 1 ? "y" : "ies" - print(iostr, " $(ndeps) dependenc$(plural) successfully precompiled in $(seconds_elapsed) seconds") - if n_already_precomp > 0 || !isempty(circular_deps) || !isempty(skipped_deps) + plural = length(configs) > 1 ? "dependency configurations" : ndeps == 1 ? "dependency" : "dependencies" + print(iostr, " $(ndeps) $(plural) successfully precompiled in $(seconds_elapsed) seconds") + if n_already_precomp > 0 || !isempty(circular_deps) n_already_precomp > 0 && (print(iostr, ". $n_already_precomp already precompiled")) !isempty(circular_deps) && (print(iostr, ". $(length(circular_deps)) skipped due to circular dependency")) - !isempty(skipped_deps) && (print(iostr, ". $(length(skipped_deps)) skipped during auto due to previous errors")) print(iostr, ".") end if n_loaded > 0 - plural1 = n_loaded == 1 ? "y" : "ies" + plural1 = length(configs) > 1 ? "dependency configurations" : n_loaded == 1 ? "dependency" : "dependencies" plural2 = n_loaded == 1 ? "a different version is" : "different versions are" plural3 = n_loaded == 1 ? "" : "s" print(iostr, "\n ", color_string(string(n_loaded), Base.warn_color()), - " dependenc$(plural1) precompiled but $(plural2) currently loaded. Restart julia to access the new version$(plural3)" + " $(plural1) precompiled but $(plural2) currently loaded. Restart julia to access the new version$(plural3)" ) end if !isempty(precomperr_deps) - pluralpc = length(precomperr_deps) == 1 ? "y" : "ies" + pluralpc = length(configs) > 1 ? "dependency configurations" : precomperr_deps == 1 ? "dependency" : "dependencies" print(iostr, "\n ", color_string(string(length(precomperr_deps)), Base.warn_color()), - " dependenc$(pluralpc) failed but may be precompilable after restarting julia" + " $(pluralpc) failed but may be precompilable after restarting julia" ) end end @@ -858,13 +891,14 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false plural1 = length(std_outputs) == 1 ? "y" : "ies" plural2 = length(std_outputs) == 1 ? "" : "s" print(iostr, "\n ", color_string("$(length(std_outputs))", Base.warn_color()), " dependenc$(plural1) had output during precompilation:") - for (pkgid, err) in std_outputs - err = if pkgid == pkg_liveprinted + for (pkg_config, err) in std_outputs + pkg, config = pkg_config + err = if pkg == pkg_liveprinted "[Output was shown above]" else join(split(strip(err), "\n"), color_string("\n│ ", Base.warn_color())) end - name = haskey(exts, pkgid) ? string(exts[pkgid], " → ", pkgid.name) : pkgid.name + name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name print(iostr, color_string("\n┌ ", Base.warn_color()), name, color_string("\n│ ", Base.warn_color()), err, color_string("\n└ ", Base.warn_color())) end end @@ -920,14 +954,16 @@ function _color_string(cstr::String, col::Union{Int64, Symbol}, hascolor) end # Can be merged with `maybe_cachefile_lock` in loading? -function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg::Base.PkgId, pkgspidlocked::Dict{Base.PkgId,String}, hascolor, cacheflags) +function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg_config, pkgspidlocked, hascolor) + pkg, config = pkg_config + flags, cacheflags = config FileWatching = Base.loaded_modules[Base.PkgId(Base.UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"), "FileWatching")] stale_age = Base.compilecache_pidlock_stale_age - pidfile = Base.compilecache_pidfile_path(pkg) + pidfile = Base.compilecache_pidfile_path(pkg, flags=cacheflags) cachefile = FileWatching.trymkpidlock(f, pidfile; stale_age) if cachefile === false pid, hostname, age = FileWatching.Pidfile.parse_pidfile(pidfile) - pkgspidlocked[pkg] = if isempty(hostname) || hostname == gethostname() + pkgspidlocked[pkg_config] = if isempty(hostname) || hostname == gethostname() if pid == getpid() "an async task in this process (pidfile: $pidfile)" else @@ -937,7 +973,7 @@ function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLo "another machine (hostname: $hostname, pid: $pid, pidfile: $pidfile)" end !fancyprint && lock(print_lock) do - println(io, " ", pkg.name, _color_string(" Being precompiled by $(pkgspidlocked[pkg])", Base.info_color(), hascolor)) + println(io, " ", pkg.name, _color_string(" Being precompiled by $(pkgspidlocked[pkg_config])", Base.info_color(), hascolor)) end # wait until the lock is available FileWatching.mkpidlock(pidfile; stale_age) do @@ -945,7 +981,7 @@ function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLo if Base.isprecompiled(pkg; ignore_loaded=true, flags=cacheflags) # don't use caches for this as the env state will have changed return nothing # returning nothing indicates a process waited for another else - delete!(pkgspidlocked, pkg) + delete!(pkgspidlocked, pkg_config) return f() # precompile end end diff --git a/base/util.jl b/base/util.jl index c91f04e8baf96..5e86f026f8f9a 100644 --- a/base/util.jl +++ b/base/util.jl @@ -687,6 +687,7 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), pathsep = Sys.iswindows() ? ";" : ":" ENV2["JULIA_DEPOT_PATH"] = string(mktempdir(; cleanup = true), pathsep) # make sure the default depots can be loaded ENV2["JULIA_LOAD_PATH"] = string("@", pathsep, "@stdlib") + ENV2["JULIA_TESTS"] = "true" delete!(ENV2, "JULIA_PROJECT") try run(setenv(`$(julia_cmd()) $(joinpath(Sys.BINDIR, diff --git a/pkgimage.mk b/pkgimage.mk index c9de49d2f8421..485e27996bb44 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -7,146 +7,31 @@ include $(JULIAHOME)/stdlib/stdlib.mk # set some influential environment variables export JULIA_DEPOT_PATH := $(build_prefix)/share/julia -export JULIA_LOAD_PATH := @stdlib +export JULIA_LOAD_PATH := @stdlib:$(JULIAHOME)/stdlib unexport JULIA_PROJECT := unexport JULIA_BINDIR := export JULIA_FALLBACK_REPL := true default: release -release: all-release -debug: all-debug +release: $(BUILDDIR)/stdlib/release.image +debug: $(BUILDDIR)/stdlib/debug.image all: release debug -$(JULIA_DEPOT_PATH): +$(JULIA_DEPOT_PATH)/compiled: mkdir -p $@ print-depot-path: @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e '@show Base.DEPOT_PATH') -all-release: $(addprefix cache-release-, $(STDLIBS)) -all-debug: $(addprefix cache-debug-, $(STDLIBS)) +$(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(JULIA_DEPOT_PATH)/compiled + export JULIA_CPU_TARGET="$(JULIA_CPU_TARGET)" + @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.Precompilation.precompilepkgs(;configs=[``=>Base.CacheFlags(), `--check-bounds=yes`=>Base.CacheFlags(;check_bounds=1)])') + touch $@ -define stdlib_builder -ifneq ($(filter $(1),$(INDEPENDENT_STDLIBS) $(PKG_EXTS)),) -# Define target-specific export for `JULIA_CPU_TARGET` -$$(BUILDDIR)/stdlib/$1.release.image: export JULIA_CPU_TARGET=$(JULIA_CPU_TARGET) -$$(BUILDDIR)/stdlib/$1.debug.image: export JULIA_CPU_TARGET=$(JULIA_CPU_TARGET) -ifneq ($(filter $(1),$(INDEPENDENT_STDLIBS)),) -$$(BUILDDIR)/stdlib/$1.release.image: $$($1_SRCS) $$(addsuffix .release.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys.$(SHLIB_EXT) - @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') - @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))') - touch $$@ -$$(BUILDDIR)/stdlib/$1.debug.image: $$($1_SRCS) $$(addsuffix .debug.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) - @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') - @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))') - touch $$@ -endif -ifneq ($(filter $(1),$(PKG_EXTS)),) -# This is weird. It set up for multiple Pkg exts because that suits the structure here better -# but it hard codes the deps and `import REPL, Pkg` -$$(BUILDDIR)/stdlib/REPLExt.release.image: $$(REPLExt_SRCS) $$(BUILDDIR)/stdlib/Pkg.release.image $$(BUILDDIR)/stdlib/REPL.release.image - @$$(call PRINT_JULIA, $$(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'using REPL; using Pkg') - @$$(call PRINT_JULIA, $$(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e 'using REPL; using Pkg') - touch $$@ -$$(BUILDDIR)/stdlib/REPLExt.debug.image: $$(REPLExt_SRCS) $(BUILDDIR)/stdlib/Pkg.debug.image $$(BUILDDIR)/stdlib/REPL.debug.image - @$$(call PRINT_JULIA, $$(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'using REPL; using Pkg') - @$$(call PRINT_JULIA, $$(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e 'using REPL; using Pkg') - touch $$@ -endif -else -ifneq ($(filter $(1),$(STDLIBS_WITHIN_SYSIMG)),) -$$(BUILDDIR)/stdlib/$1.release.image: - touch $$@ -$$(BUILDDIR)/stdlib/$1.debug.image: - touch $$@ -else -$$(error $(1) neither in STDLIBS_WITHIN_SYSIMG nor INDEPENDENT_STDLIBS) -endif -endif -cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image -cache-debug-$1: $$(BUILDDIR)/stdlib/$1.debug.image -.SECONDARY: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.debug.image -endef +$(BUILDDIR)/stdlib/release.image: $(build_private_libdir)/sys.$(SHLIB_EXT) +$(BUILDDIR)/stdlib/debug.image: $(build_private_libdir)/sys-debug.$(SHLIB_EXT) -# Note: you can check for the correctness of this tree by running `JULIA_DEBUG=nested_precomp make` and looking -# out for `Debug: Nested precompilation` logs. - -# no dependencies -$(eval $(call stdlib_builder,MozillaCACerts_jll,)) -$(eval $(call stdlib_builder,ArgTools,)) -$(eval $(call stdlib_builder,Artifacts,)) -$(eval $(call stdlib_builder,Base64,)) -$(eval $(call stdlib_builder,CRC32c,)) -$(eval $(call stdlib_builder,FileWatching,)) -$(eval $(call stdlib_builder,Libdl,)) -$(eval $(call stdlib_builder,Mmap,)) -$(eval $(call stdlib_builder,NetworkOptions,)) -$(eval $(call stdlib_builder,SHA,)) -$(eval $(call stdlib_builder,Serialization,)) -$(eval $(call stdlib_builder,Sockets,)) -$(eval $(call stdlib_builder,Unicode,)) -$(eval $(call stdlib_builder,Profile,)) -$(eval $(call stdlib_builder,StyledStrings,)) -$(eval $(call stdlib_builder,SuiteSparse_jll,)) - -# 1-depth packages -$(eval $(call stdlib_builder,GMP_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,LLVMLibUnwind_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,LibUV_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,LibUnwind_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,MbedTLS_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,nghttp2_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,OpenLibm_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,PCRE2_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,Zlib_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,dSFMT_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,libLLVM_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,libblastrampoline_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,p7zip_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,OpenBLAS_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,Markdown,Base64)) -$(eval $(call stdlib_builder,Printf,Unicode)) -$(eval $(call stdlib_builder,Random,SHA)) -$(eval $(call stdlib_builder,Logging,StyledStrings)) -$(eval $(call stdlib_builder,Tar,ArgTools,SHA)) -$(eval $(call stdlib_builder,DelimitedFiles,Mmap)) -$(eval $(call stdlib_builder,JuliaSyntaxHighlighting,StyledStrings)) - -# 2-depth packages -$(eval $(call stdlib_builder,LLD_jll,Zlib_jll libLLVM_jll Artifacts Libdl)) -$(eval $(call stdlib_builder,LibSSH2_jll,Artifacts Libdl MbedTLS_jll)) -$(eval $(call stdlib_builder,MPFR_jll,Artifacts Libdl GMP_jll)) -$(eval $(call stdlib_builder,LinearAlgebra,Libdl libblastrampoline_jll OpenBLAS_jll)) -$(eval $(call stdlib_builder,Dates,Printf)) -$(eval $(call stdlib_builder,Distributed,Random Serialization Sockets)) -$(eval $(call stdlib_builder,Future,Random)) -$(eval $(call stdlib_builder,UUIDs,Random SHA)) -$(eval $(call stdlib_builder,InteractiveUtils,Markdown)) - - # 3-depth packages -$(eval $(call stdlib_builder,LibGit2_jll,MbedTLS_jll LibSSH2_jll Artifacts Libdl)) -$(eval $(call stdlib_builder,LibCURL_jll,LibSSH2_jll nghttp2_jll MbedTLS_jll Zlib_jll Artifacts Libdl)) -$(eval $(call stdlib_builder,REPL,InteractiveUtils Markdown Sockets StyledStrings Unicode)) -$(eval $(call stdlib_builder,SharedArrays,Distributed Mmap Random Serialization)) -$(eval $(call stdlib_builder,TOML,Dates)) -$(eval $(call stdlib_builder,Test,Logging Random Serialization InteractiveUtils)) - -# 4-depth packages -$(eval $(call stdlib_builder,LibGit2,LibGit2_jll NetworkOptions Printf SHA Base64)) -$(eval $(call stdlib_builder,LibCURL,LibCURL_jll MozillaCACerts_jll)) - -# 5-depth packages -$(eval $(call stdlib_builder,Downloads,ArgTools FileWatching LibCURL NetworkOptions)) - -# 6-depth packages -$(eval $(call stdlib_builder,Pkg, Artifacts Dates Downloads FileWatching LibGit2 Libdl\ - Logging Markdown Printf REPL Random SHA Serialization\ - TOML Tar UUIDs p7zip_jll)) - -# 7-depth packages -$(eval $(call stdlib_builder,LazyArtifacts,Artifacts Pkg)) -$(eval $(call stdlib_builder,REPLExt,Pkg REPL)) - -$(eval $(call stdlib_builder,SparseArrays,Libdl LinearAlgebra Random Serialization SuiteSparse_jll)) -$(eval $(call stdlib_builder,Statistics,LinearAlgebra SparseArrays)) +clean: + rm -rf $(JULIA_DEPOT_PATH)/compiled + rm -f $(BUILDDIR)/stdlib/*.image diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml new file mode 100644 index 0000000000000..f2ae43f231db5 --- /dev/null +++ b/stdlib/Manifest.toml @@ -0,0 +1,300 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.0-DEV" +manifest_format = "2.0" +project_hash = "13f2dd600364a1e8b659dc5796bf185b37d1c95d" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.CRC32c]] +uuid = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" +version = "1.11.0" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.1.0+0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" + +[[deps.GMP_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" +version = "6.3.0+0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "dc6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.11.0" + +[[deps.LLD_jll]] +deps = ["Artifacts", "Libdl", "Zlib_jll", "libLLVM_jll"] +uuid = "d55e3150-da41-5e91-b323-ecfd1eec6109" +version = "16.0.6+4" + +[[deps.LLVMLibUnwind_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "47c5dbc3-30ba-59ef-96a6-123e260183d9" +version = "12.0.1+0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.6.0+0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.7.2+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.LibUV_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "183b4373-6708-53ba-ad28-60e28bb38547" +version = "2.0.1+15" + +[[deps.LibUnwind_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "745a5e78-f969-53e9-954f-d19f2f74f4e3" +version = "1.7.2+2" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.11.0" + +[[deps.Logging]] +deps = ["StyledStrings"] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.MPFR_jll]] +deps = ["Artifacts", "GMP_jll", "Libdl"] +uuid = "3a97d323-0669-5f0c-9066-3539efd106a3" +version = "4.2.0+1" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.6+0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.12.12" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.26+2" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+2" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.42.0+1" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.11.0" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.Profile]] +uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" +version = "1.11.0" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" +version = "1.11.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.11.0" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.6.0+0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.dSFMT_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05ff407c-b0c1-5878-9df8-858cc2e60c36" +version = "2.2.5+0" + +[[deps.libLLVM_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8f36deef-c2a5-5394-99ed-8e07531fb29a" +version = "16.0.6+4" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+1" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.59.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" diff --git a/stdlib/Project.toml b/stdlib/Project.toml new file mode 100644 index 0000000000000..cc7ba99dd4e4f --- /dev/null +++ b/stdlib/Project.toml @@ -0,0 +1,61 @@ +[deps] +ArgTools = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +Future = "9fa8497b-333b-5362-9e8d-4d0656e87820" +GMP_jll = "781609d7-10c4-51f6-84f2-b8444358ff6d" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +JuliaSyntaxHighlighting = "dc6e5ff7-fb65-4e79-a425-ec3bc9c03011" +LLD_jll = "d55e3150-da41-5e91-b323-ecfd1eec6109" +LLVMLibUnwind_jll = "47c5dbc3-30ba-59ef-96a6-123e260183d9" +LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +LibCURL = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +LibCURL_jll = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" +LibGit2_jll = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +LibSSH2_jll = "29816b5a-b9ab-546f-933c-edad1886dfa8" +LibUV_jll = "183b4373-6708-53ba-ad28-60e28bb38547" +LibUnwind_jll = "745a5e78-f969-53e9-954f-d19f2f74f4e3" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +MPFR_jll = "3a97d323-0669-5f0c-9066-3539efd106a3" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" +MbedTLS_jll = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +Mmap = "a63ad114-7e13-5084-954f-fe012c677804" +MozillaCACerts_jll = "14a3606d-f60d-562e-9121-12d972cd8159" +NetworkOptions = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +OpenBLAS_jll = "4536629a-c528-5b80-bd46-f80d51c5b363" +OpenLibm_jll = "05823500-19ac-5b8b-9628-191a04bc5112" +PCRE2_jll = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +SharedArrays = "1a1011a3-84de-559e-8e89-a11a2f7dc383" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +StyledStrings = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +SuiteSparse_jll = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" +dSFMT_jll = "05ff407c-b0c1-5878-9df8-858cc2e60c36" +libLLVM_jll = "8f36deef-c2a5-5394-99ed-8e07531fb29a" +libblastrampoline_jll = "8e850b90-86db-534c-a0d3-1478176c7d93" +nghttp2_jll = "8e850ede-7688-5339-a07c-302acd2aaf8d" +p7zip_jll = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/stdlib/stdlib.mk b/stdlib/stdlib.mk index b23754f3a3cac..b79059d3368b1 100644 --- a/stdlib/stdlib.mk +++ b/stdlib/stdlib.mk @@ -11,24 +11,19 @@ INDEPENDENT_STDLIBS := \ LibCURL_jll LibSSH2_jll LibGit2_jll nghttp2_jll MozillaCACerts_jll MbedTLS_jll \ MPFR_jll OpenLibm_jll PCRE2_jll p7zip_jll Zlib_jll -PKG_EXTS := \ - REPLExt - -STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS) $(PKG_EXTS) +STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS) VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) -SYSIMG_STDLIB_SRCS = +SYSIMG_STDLIBS_SRCS = +INDEPENDENT_STDLIBS_SRCS = define STDLIB_srcs -ifneq ($(filter $(1),$(PKG_EXTS)),) - $1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/Pkg/ext/$1 -name \*.jl) \ - $$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/Pkg) $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/Pkg/Project.toml -else - $1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/src -name \*.jl) \ - $$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/$1) $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/Project.toml -endif +$1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/src -name \*.jl) \ +$$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/$1) $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/Project.toml ifneq ($(filter $(1),$(STDLIBS_WITHIN_SYSIMG)),) - SYSIMG_STDLIB_SRCS += $$($1_SRCS) + SYSIMG_STDLIBS_SRCS += $$($1_SRCS) +else + INDEPENDENT_STDLIBS_SRCS += $$($1_SRCS) endif endef diff --git a/sysimage.mk b/sysimage.mk index e6fa54be5f186..b5f47be0160a4 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -57,7 +57,7 @@ COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl) # sort these to remove duplicates BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \ $(shell find $(BUILDROOT)/base -name \*.jl -and -not -name sysimg.jl)) -STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(SYSIMG_STDLIB_SRCS) +STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(SYSIMG_STDLIBS_SRCS) RELBUILDROOT := $(call rel_path,$(JULIAHOME)/base,$(BUILDROOT)/base)/ # <-- make sure this always has a trailing slash $(build_private_libdir)/corecompiler.ji: $(COMPILER_SRCS) From e6e8a1d4ca24f5f24ab64c407f25659892c66cbd Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 9 Mar 2024 08:28:55 -0500 Subject: [PATCH 35/55] Fix bootstrap Base precompile in cross compile configuration (#53671) (cherry picked from commit 18a2e70636b6cd69c67f9e83fd34ddb8fb0649ce) --- pkgimage.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgimage.mk b/pkgimage.mk index 485e27996bb44..740b9760cab48 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -6,8 +6,8 @@ include $(JULIAHOME)/stdlib/stdlib.mk # set some influential environment variables -export JULIA_DEPOT_PATH := $(build_prefix)/share/julia -export JULIA_LOAD_PATH := @stdlib:$(JULIAHOME)/stdlib +export JULIA_DEPOT_PATH := $(shell echo $(call cygpath_w,$(build_prefix)/share/julia)) +export JULIA_LOAD_PATH := @stdlib$(PATHSEP)$(shell echo $(call cygpath_w,$(JULIAHOME)/stdlib)) unexport JULIA_PROJECT := unexport JULIA_BINDIR := From fcadc061922ac6facf014282e9aee9a1cf42e2c3 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 9 Mar 2024 09:58:25 -0500 Subject: [PATCH 36/55] Load Pkg if not already to reinstate missing package add prompt (#52125) (cherry picked from commit 0d029eaf7ddc594a59cae90096419c4f66d45855) --- stdlib/REPL/src/REPL.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 840e6daaf1909..43181485183d2 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -215,9 +215,7 @@ function eval_user_input(@nospecialize(ast), backend::REPLBackend, mod::Module) put!(backend.response_channel, Pair{Any, Bool}(lasterr, true)) else backend.in_eval = true - if !isempty(install_packages_hooks) - check_for_missing_packages_and_run_hooks(ast) - end + check_for_missing_packages_and_run_hooks(ast) for xf in backend.ast_transforms ast = Base.invokelatest(xf, ast) end @@ -244,6 +242,7 @@ function check_for_missing_packages_and_run_hooks(ast) mods = modules_to_be_loaded(ast) filter!(mod -> isnothing(Base.identify_package(String(mod))), mods) # keep missing modules if !isempty(mods) + isempty(install_packages_hooks) && Base.require_stdlib(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) for f in install_packages_hooks Base.invokelatest(f, mods) && return end @@ -1651,7 +1650,6 @@ function __current_ast_transforms(backend) end end - function numbered_prompt!(repl::LineEditREPL=Base.active_repl, backend=nothing) n = Ref{Int}(0) set_prompt(repl, n) From 8c9015ff08631da0943111392efcdc23e43117b1 Mon Sep 17 00:00:00 2001 From: Frames White Date: Sun, 10 Mar 2024 02:36:14 +0800 Subject: [PATCH 37/55] Handle zero on arrays of unions of number types and missings (#53602) (cherry picked from commit fc6c618c4e199330429c05836eeb779723e7c97b) --- base/abstractarray.jl | 1 + test/abstractarray.jl | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index c9307c8c0f70f..749ca98a587b0 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1218,6 +1218,7 @@ end copymutable(itr) = collect(itr) zero(x::AbstractArray{T}) where {T<:Number} = fill!(similar(x, typeof(zero(T))), zero(T)) +zero(x::AbstractArray{S}) where {S<:Union{Missing, Number}} = fill!(similar(x, typeof(zero(S))), zero(S)) zero(x::AbstractArray) = map(zero, x) ## iteration support for arrays by iterating over `eachindex` in the array ## diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 8392d533261b3..15f8ed638bcc5 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -1972,6 +1972,16 @@ end @test zero([[2,2], [3,3,3]]) isa Vector{Vector{Int}} @test zero([[2,2], [3,3,3]]) == [[0,0], [0, 0, 0]] + + + @test zero(Union{Float64, Missing}[missing]) == [0.0] + struct CustomNumber <: Number + val::Float64 + end + Base.zero(::Type{CustomNumber}) = CustomNumber(0.0) + @test zero([CustomNumber(5.0)]) == [CustomNumber(0.0)] + @test zero(Union{CustomNumber, Missing}[missing]) == [CustomNumber(0.0)] + @test zero(Vector{Union{CustomNumber, Missing}}(undef, 1)) == [CustomNumber(0.0)] end @testset "`_prechecked_iterate` optimization" begin From aa90ae7cef221218507e28ecaba86e498e5c83f6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 9 Mar 2024 13:40:54 -0500 Subject: [PATCH 38/55] permit NamedTuple{<:Any, Union{}} to be created (#53516) (cherry picked from commit e9c84c809f6369907f4087e8d823f13c92a93c35) --- base/boot.jl | 4 ++-- base/compiler/abstractinterpretation.jl | 2 +- base/namedtuple.jl | 19 ++++++++----------- src/builtins.c | 2 ++ src/jltypes.c | 17 +++++++++++------ test/namedtuple.jl | 15 +++++++++++++++ 6 files changed, 39 insertions(+), 20 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index f8ed86b0fb6b7..5980818a2047b 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -727,8 +727,8 @@ eval(Core, :(NamedTuple{names}(args::Tuple) where {names} = using .Intrinsics: sle_int, add_int -eval(Core, :(NamedTuple{names,T}(args::T) where {names, T <: Tuple} = - $(Expr(:splatnew, :(NamedTuple{names,T}), :args)))) +eval(Core, :((NT::Type{NamedTuple{names,T}})(args::T) where {names, T <: Tuple} = + $(Expr(:splatnew, :NT, :args)))) # constructors for built-in types diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 21303f527e3f2..e79c7188da868 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -2686,7 +2686,7 @@ function refine_partial_type(@nospecialize t) # if the first/second parameter of `NamedTuple` is known to be empty, # the second/first argument should also be empty tuple type, # so refine it here - return Const(NamedTuple()) + return Const((;)) end return t end diff --git a/base/namedtuple.jl b/base/namedtuple.jl index cbcade94f1c0d..98192480db9dd 100644 --- a/base/namedtuple.jl +++ b/base/namedtuple.jl @@ -112,24 +112,24 @@ Core.NamedTuple if nameof(@__MODULE__) === :Base -@eval function NamedTuple{names,T}(args::Tuple) where {names, T <: Tuple} +@eval function (NT::Type{NamedTuple{names,T}})(args::Tuple) where {names, T <: Tuple} if length(args) != length(names::Tuple) throw(ArgumentError("Wrong number of arguments to named tuple constructor.")) end # Note T(args) might not return something of type T; e.g. # Tuple{Type{Float64}}((Float64,)) returns a Tuple{DataType} - $(Expr(:splatnew, :(NamedTuple{names,T}), :(T(args)))) + $(Expr(:splatnew, :NT, :(T(args)))) end -function NamedTuple{names, T}(nt::NamedTuple) where {names, T <: Tuple} +function (NT::Type{NamedTuple{names, T}})(nt::NamedTuple) where {names, T <: Tuple} if @generated - Expr(:new, :(NamedTuple{names, T}), - Any[ :(let Tn = fieldtype(T, $n), + Expr(:new, :NT, + Any[ :(let Tn = fieldtype(NT, $n), ntn = getfield(nt, $(QuoteNode(names[n]))) ntn isa Tn ? ntn : convert(Tn, ntn) end) for n in 1:length(names) ]...) else - NamedTuple{names, T}(map(Fix1(getfield, nt), names)) + NT(map(Fix1(getfield, nt), names)) end end @@ -145,14 +145,11 @@ function NamedTuple{names}(nt::NamedTuple) where {names} end end -NamedTuple{names, T}(itr) where {names, T <: Tuple} = NamedTuple{names, T}(T(itr)) -NamedTuple{names}(itr) where {names} = NamedTuple{names}(Tuple(itr)) +(NT::Type{NamedTuple{names, T}})(itr) where {names, T <: Tuple} = NT(T(itr)) +(NT::Type{NamedTuple{names}})(itr) where {names} = NT(Tuple(itr)) NamedTuple(itr) = (; itr...) -# avoids invalidating Union{}(...) -NamedTuple{names, Union{}}(itr::Tuple) where {names} = throw(MethodError(NamedTuple{names, Union{}}, (itr,))) - end # if Base # Like NamedTuple{names, T} as a constructor, but omits the additional diff --git a/src/builtins.c b/src/builtins.c index 29aec53ae5a40..c26d528cb3ee4 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1219,6 +1219,8 @@ static jl_value_t *get_fieldtype(jl_value_t *t, jl_value_t *f, int dothrow) tt = ((jl_tvar_t*)tt)->ub; if (tt == (jl_value_t*)jl_any_type) return (jl_value_t*)jl_any_type; + if (tt == (jl_value_t*)jl_bottom_type) + return (jl_value_t*)jl_bottom_type; JL_GC_PUSH1(&f); if (jl_is_symbol(f)) f = jl_box_long(field_index+1); diff --git a/src/jltypes.c b/src/jltypes.c index e048b5a0f8dfd..ef58deef2c634 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2141,15 +2141,20 @@ static jl_value_t *inst_datatype_inner(jl_datatype_t *dt, jl_svec_t *p, jl_value jl_errorf("duplicate field name in NamedTuple: \"%s\" is not unique", jl_symbol_name((jl_sym_t*)ni)); } } - if (!jl_is_datatype(values_tt)) - jl_error("NamedTuple field type must be a tuple type"); - if (jl_is_va_tuple((jl_datatype_t*)values_tt) || jl_nparams(values_tt) != nf) - jl_error("NamedTuple names and field types must have matching lengths"); - ndt->types = ((jl_datatype_t*)values_tt)->parameters; + if (values_tt == jl_bottom_type && nf > 0) { + ndt->types = jl_svec_fill(nf, jl_bottom_type); + } + else { + if (!jl_is_datatype(values_tt)) + jl_error("NamedTuple field type must be a tuple datatype"); + if (jl_is_va_tuple((jl_datatype_t*)values_tt) || jl_nparams(values_tt) != nf) + jl_error("NamedTuple names and field types must have matching lengths"); + ndt->types = ((jl_datatype_t*)values_tt)->parameters; + } jl_gc_wb(ndt, ndt->types); } else { - ndt->types = jl_emptysvec; // XXX: this is essentially always false + ndt->types = jl_emptysvec; // XXX: this is essentially always incorrect } } else if (tn == jl_genericmemoryref_typename || tn == jl_genericmemory_typename) { diff --git a/test/namedtuple.jl b/test/namedtuple.jl index 9c16d24b0c575..48aa8ea4a2591 100644 --- a/test/namedtuple.jl +++ b/test/namedtuple.jl @@ -437,3 +437,18 @@ let c = (a=1, b=2), d = (b=3, c=(d=1,)) @test @inferred(mergewith51009((x,y)->y, c, d)) === (a = 1, b = 3, c = (d = 1,)) end + +@test_throws ErrorException NamedTuple{(), Union{}} +for NT in (NamedTuple{(:a, :b), Union{}}, NamedTuple{(:a, :b), T} where T<:Union{}) + @test fieldtype(NT, 1) == Union{} + @test fieldtype(NT, :b) == Union{} + @test_throws ErrorException fieldtype(NT, :c) + @test_throws BoundsError fieldtype(NT, 0) + @test_throws BoundsError fieldtype(NT, 3) + @test Base.return_types((Type{NT},)) do NT; fieldtype(NT, :a); end == Any[Type{Union{}}] + @test fieldtype(NamedTuple{<:Any, Union{}}, 1) == Union{} +end +let NT = NamedTuple{<:Any, Union{}} + @test fieldtype(NT, 100) == Union{} + @test only(Base.return_types((Type{NT},)) do NT; fieldtype(NT, 100); end) >: Type{Union{}} +end From 760f3681a9006db8ff8154ca6f9726ffc76477b8 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Sat, 9 Mar 2024 17:22:20 -0500 Subject: [PATCH 39/55] Bump CSL to 1.1.1 to fix libgomp bug (#53643) Resolves https://github.com/JuliaLang/julia/issues/53363 (cherry picked from commit 5c7d24493ebab184d4517ca556314524f4fcb47f) --- deps/checksums/compilersupportlibraries | 184 +++++++++--------- .../CompilerSupportLibraries_jll/Project.toml | 2 +- 2 files changed, 93 insertions(+), 93 deletions(-) diff --git a/deps/checksums/compilersupportlibraries b/deps/checksums/compilersupportlibraries index 2dcfecfb56b26..48843f21c0feb 100644 --- a/deps/checksums/compilersupportlibraries +++ b/deps/checksums/compilersupportlibraries @@ -1,92 +1,92 @@ -CompilerSupportLibraries.v1.1.0+0.aarch64-apple-darwin-libgfortran5.tar.gz/md5/20ebaad57850393b6ac9fa924e511fe4 -CompilerSupportLibraries.v1.1.0+0.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/020de4d8b0ff6bedbadaa305ff8445e6849f12053762ea4aa68412d1ec763dbd86f479587a2fbb862487f1feb04d976c38099ddf3887817a3d32b3f029cf85b1 -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-gnu-libgfortran3.tar.gz/md5/e084a4374be45ba52279682c640449bc -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-gnu-libgfortran3.tar.gz/sha512/d4aedf5c08e13fd9596476330f69e374af64373f7bca0e4df6cbb4d1710d695dce23f2655ee368c3df2049b7b0ca1848c9e01a437fadc0eb08937c6a7cdf2a27 -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-gnu-libgfortran4.tar.gz/md5/6b3975f25be16ea1370ef0bf353ac752 -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-gnu-libgfortran4.tar.gz/sha512/d6298517be1ce350a61d1c1a1bf9b27a541382aa8ccf3e86eeadd7c47e2fe88facd17139a3878adb939df869a330264a942d280e3468d53b61325df7e31daaad -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-gnu-libgfortran5.tar.gz/md5/7134b79b71059d4da79224df1ca0853e -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/e66590a37e756ff33a84514a7ca2bbe2e1517f3e901bc66e40139e8a318d6cd8e329e0c2a7c557ea39e6db2f56802d485ab81b87be1373717b78474b1c7bf7d7 -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-musl-libgfortran3.tar.gz/md5/fd37789f5745a17cc9a85902cebf4698 -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-musl-libgfortran3.tar.gz/sha512/9ff4e9be39f115af2e5bb6a5c88b3940f15a010952cebf39da22e7a5c6744be2f905bebccba092db0a89cf82e8c0e1a3e61b74d4204d2a6648b5469f3ccb0d12 -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-musl-libgfortran4.tar.gz/md5/e9286ae9299c57d5df7b795997b4adf5 -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-musl-libgfortran4.tar.gz/sha512/ea64858481095e0374be330aa2ac84b394bc3e5351b9326137c9cd5d15e6bec47d6e5f672a216572dcb80c3aa6fcb08950cc10157c264f429a93c235028d79a4 -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-musl-libgfortran5.tar.gz/md5/b19c6cbc5b2a62ea76dea64b0f8ae488 -CompilerSupportLibraries.v1.1.0+0.aarch64-linux-musl-libgfortran5.tar.gz/sha512/090d659f7e4a7034117e2bb2dcad0ef544cdca898bf032222cdb81d32af6e6528be842d2cf55839fe397c2ace05dd4ce920cf98cc96324ae18832016516e6cc3 -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/md5/26fb41031e1b797373aea7a7c4d7be3c -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/sha512/8fd0b6c990681789caec528067b4bcb9661f9c0a5e0268927d4e88565fa7005db3b592fb8e7830cf32b3fb4ce54d6db747dfde896f93bd38f65b7a1290a2399a -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/md5/6ed3f3e94f662177c3cf3c3734a5c1ec -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/sha512/b8165630cced0f7880cb6fd6263cf39bbbbda668eccc94219720078a85a641c3b1b20648960041aa3a51108ab6df087b909c572d0690aacf8b99dc5496ff7db6 -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/86060cbbe966a59f18f92a9b2fab95d4 -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/0aa0ca0ff3a4d541c7a9599ca1adae7391fdd3fa841f3055ecb8635096d0d95a0763758d7533c887b38a655af55174dfcb63f470147b28a256b75a85c8e47801 -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/md5/4acf6f8929fb8ec9fdb8a0f1af06260d -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/sha512/3b2e0a5f62bd93434d07848c3045479a1a05bd8589dc976a5680e13805db5adcd9abdcca82edee7b28b4c4a9413ce795784a8a0f0a8fb7346a439322c27c96d9 -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/md5/a75a927c3e14bee6dca29b4907def681 -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/sha512/7853fd187f9289a8282d34112b5277bad13abe9dd9b6c796498db2f1a080b2c81faa6119df9ececd09725a019bf99706894765c9c20f618e359adc153c3181a2 -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/ba8545cc20e6c602a0526a3b1fc1d2f1 -CompilerSupportLibraries.v1.1.0+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/7a7f3a7761deb068efc00ffc5d4bf4df365cb27674ce73abbe2305b678285161f1526f4facbe27fc11076d99b2079976507f78f5b463bd9057ed008e9d52f9cf -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/md5/26fb41031e1b797373aea7a7c4d7be3c -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/sha512/8fd0b6c990681789caec528067b4bcb9661f9c0a5e0268927d4e88565fa7005db3b592fb8e7830cf32b3fb4ce54d6db747dfde896f93bd38f65b7a1290a2399a -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/md5/6ed3f3e94f662177c3cf3c3734a5c1ec -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/sha512/b8165630cced0f7880cb6fd6263cf39bbbbda668eccc94219720078a85a641c3b1b20648960041aa3a51108ab6df087b909c572d0690aacf8b99dc5496ff7db6 -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/86060cbbe966a59f18f92a9b2fab95d4 -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/0aa0ca0ff3a4d541c7a9599ca1adae7391fdd3fa841f3055ecb8635096d0d95a0763758d7533c887b38a655af55174dfcb63f470147b28a256b75a85c8e47801 -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/md5/4acf6f8929fb8ec9fdb8a0f1af06260d -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/sha512/3b2e0a5f62bd93434d07848c3045479a1a05bd8589dc976a5680e13805db5adcd9abdcca82edee7b28b4c4a9413ce795784a8a0f0a8fb7346a439322c27c96d9 -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/md5/a75a927c3e14bee6dca29b4907def681 -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/sha512/7853fd187f9289a8282d34112b5277bad13abe9dd9b6c796498db2f1a080b2c81faa6119df9ececd09725a019bf99706894765c9c20f618e359adc153c3181a2 -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/ba8545cc20e6c602a0526a3b1fc1d2f1 -CompilerSupportLibraries.v1.1.0+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/7a7f3a7761deb068efc00ffc5d4bf4df365cb27674ce73abbe2305b678285161f1526f4facbe27fc11076d99b2079976507f78f5b463bd9057ed008e9d52f9cf -CompilerSupportLibraries.v1.1.0+0.i686-linux-gnu-libgfortran3.tar.gz/md5/39dc387fd58ef02c461c7906ceb110e3 -CompilerSupportLibraries.v1.1.0+0.i686-linux-gnu-libgfortran3.tar.gz/sha512/1296ac707fdad620c65256686523f2b027c8359f54d1f8354ef5d1ba514992c7269aad26b706575509b5e29d0ad3dec1c7d32fe3bcff0d723d6a4890819eca46 -CompilerSupportLibraries.v1.1.0+0.i686-linux-gnu-libgfortran4.tar.gz/md5/21a76d54d875ef09db2cdce77d328c2e -CompilerSupportLibraries.v1.1.0+0.i686-linux-gnu-libgfortran4.tar.gz/sha512/9c6bf15338ffbc7113c536e145e53bfaa693007b971f83ee2db820d7d54018bd1cfdbedb6bbce000ee7aaadad1561e91f5ac0e0519bbfccbc3bc57fdfc0eb7e7 -CompilerSupportLibraries.v1.1.0+0.i686-linux-gnu-libgfortran5.tar.gz/md5/f028f2c94f28201701ef6ba4fec9abc9 -CompilerSupportLibraries.v1.1.0+0.i686-linux-gnu-libgfortran5.tar.gz/sha512/c231af1bb0fd4f733278f883837fddf574689bbd7c4dd46cfcd1478d784cbeae1fd785d7cf9f4b0f98cda08819b63a20d5026c6beb892a188fc979b7893697bc -CompilerSupportLibraries.v1.1.0+0.i686-linux-musl-libgfortran3.tar.gz/md5/184436dc05207a653f13aae3d82a2e1b -CompilerSupportLibraries.v1.1.0+0.i686-linux-musl-libgfortran3.tar.gz/sha512/b6e1f969528a168de087f472eebd23a4daf907aa48f7c5b42c35960b1cae3e6ca8f512982d69b757f39d6dc07b46f74c84e549cb22354a2f55d1265cba7b7013 -CompilerSupportLibraries.v1.1.0+0.i686-linux-musl-libgfortran4.tar.gz/md5/545bee22cb35d1c4c1381009e72eebca -CompilerSupportLibraries.v1.1.0+0.i686-linux-musl-libgfortran4.tar.gz/sha512/78a65b9e7cda79cd648a1ae09daea970eba9d04fd5ea41bc1e37b065cf5c53974f759590292876f57c7f65139be66a6c381aa6756cdda7b36845cfed1bb7fddc -CompilerSupportLibraries.v1.1.0+0.i686-linux-musl-libgfortran5.tar.gz/md5/3f1a08601a6a7bbd4ecfa36c8f6abbd9 -CompilerSupportLibraries.v1.1.0+0.i686-linux-musl-libgfortran5.tar.gz/sha512/0e225e0a7b651f6b3fbccf760d08d66f2d8af1e329d14ef67fd3968a46905e062edcf75f60d7540f0cd7dabcd3ac9130fa0f63e198869bdc6a9aabd391652805 -CompilerSupportLibraries.v1.1.0+0.i686-w64-mingw32-libgfortran3.tar.gz/md5/9cfea65fa6c1b587d9b4b84ee64af166 -CompilerSupportLibraries.v1.1.0+0.i686-w64-mingw32-libgfortran3.tar.gz/sha512/b30e24982d4140c312759b8c26d4b72845fc1fa4d7fdf49ccfe9994f7bbf1815ed006a228f6a2185c5b8f9d596d0b04debd1d8392e705c530e5177a22c7c081d -CompilerSupportLibraries.v1.1.0+0.i686-w64-mingw32-libgfortran4.tar.gz/md5/af99905c4f054fe13842559f7201b3ad -CompilerSupportLibraries.v1.1.0+0.i686-w64-mingw32-libgfortran4.tar.gz/sha512/96513ff22dc16cc259ad392862f1765218474bff24e561f14c1e0d349a6bc433952d9b7b73236b56722fd971e0b864b178d8a9f8d9499de4595bc9857ef17a95 -CompilerSupportLibraries.v1.1.0+0.i686-w64-mingw32-libgfortran5.tar.gz/md5/1be078cd374d3b501b20d9ce679009ee -CompilerSupportLibraries.v1.1.0+0.i686-w64-mingw32-libgfortran5.tar.gz/sha512/63097283c987dc439f02d72a6f70423acd962e4da25acc04185e654c7f16a617e34ad7efabd624fd2e70119e79e4d4806f76286d36d56c353f9e53814e75d3e4 -CompilerSupportLibraries.v1.1.0+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/md5/156ae44ab4172903ad40932ca78a57ed -CompilerSupportLibraries.v1.1.0+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/sha512/e800c20342dd9886c4c3f57e92278d6d41c544adba202ef3f5a6a4f8211fbbd8fab65f169adf7320b7be8a2ea02c0aa1afedbaf0b3f9afbfb691759aaaaccc4c -CompilerSupportLibraries.v1.1.0+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/md5/cb01c02fdcbd319784034744172e1eb9 -CompilerSupportLibraries.v1.1.0+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/sha512/0ba635d39958672a0a55069521e20ca6c0f9c81a9f55c360f6043acb415709edb72bfe8d0e83c25cdf9ace8a9e9ba10e39457e234e3905c988eb95e0e0ecff3d -CompilerSupportLibraries.v1.1.0+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/f9592263c6e72228c492ed2ed216f29e -CompilerSupportLibraries.v1.1.0+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/cbe29742959906e3fe9a356991ca1f09d4d8cc2a02a9af8624b3e02b4ab59e33bc05082826f7c67c73c6b91cc8e1e5c4a0c275c21c5f8eab8b58ed942cdcb55c -CompilerSupportLibraries.v1.1.0+0.x86_64-apple-darwin-libgfortran3.tar.gz/md5/8f0db2ff4688c3f9e1337a28976d833a -CompilerSupportLibraries.v1.1.0+0.x86_64-apple-darwin-libgfortran3.tar.gz/sha512/98502e07499ad9e22147a977b1fe55320e75b6229c3993f1cd1b71e47a09ae6bf78e2341ce978ea72d33b111d09b813a332bfe8f4f6dfb669509c300fcec2561 -CompilerSupportLibraries.v1.1.0+0.x86_64-apple-darwin-libgfortran4.tar.gz/md5/980a1b8e6262c4a7b8f86b84f7234043 -CompilerSupportLibraries.v1.1.0+0.x86_64-apple-darwin-libgfortran4.tar.gz/sha512/c0755d4fbb1b6fd7755d7508d7df929feabe7e5778661397ef0205e21aa3be565b39ccc2a08ed0d958e812c0c759be68ef52de09fe92ebab6da342b309a0810d -CompilerSupportLibraries.v1.1.0+0.x86_64-apple-darwin-libgfortran5.tar.gz/md5/4b3cdb65e6114c77fd1e51da69e41afa -CompilerSupportLibraries.v1.1.0+0.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/803cb771875d94eda554bade8197b31aab988ab0c957a2f8853d82d01418be9fee7d9d4b7ef6f5b7fc8d1825ab22083a71d467eb976d5076fc5d73a9a7a30440 -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-gnu-libgfortran3.tar.gz/md5/36638a444b185954bf12169edace1914 -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-gnu-libgfortran3.tar.gz/sha512/33f657775258d0da1a57fc03c5e8ed203946944581ebf70af7b0205f9bff7fcd4f2bde5b6fa3b01659c51f106d0e6df5c7533ab8d3372c4895675854688e01dc -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-gnu-libgfortran4.tar.gz/md5/36ac52a361fd0f4be5c66572345af7a4 -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-gnu-libgfortran4.tar.gz/sha512/802bd8089bb2a3b5959a47dbade2199b46c247d0a793cbf6fcbc97b9a1dccd6d8585ac7694ae4bef1dc3ba21796ae5b53f995c8793ccd7316e8fde68ac121f83 -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-gnu-libgfortran5.tar.gz/md5/5911da90a0fc86d665aa86cba12e9d61 -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/c966936dfd272d9706aa51ed44abcb8cded899b0caa8b12ee787a0fb1569fa90a1cba89c9a9b83e05c0993facc615feb851399f4799c06956ae3064d172c964d -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-musl-libgfortran3.tar.gz/md5/5f42a52e72f0e79530d71733a93811bf -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-musl-libgfortran3.tar.gz/sha512/51078ef0e447bb181003a50b899b39a9d1ee8ecc92fc293f5a358d836ddf21d03dc44433ae28aa21fdf756c2912b2d3f1e374a5ba108c8c34552fcf32f93fd0b -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-musl-libgfortran4.tar.gz/md5/f3bbee1114cb85c266a45f64632c6911 -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-musl-libgfortran4.tar.gz/sha512/102e638f49ff0f62644f15a931c71a16b96f02f4c90d1b8bd378e0d7c54f4e8a150cdb5ffdbc3dcbafb83131bef84f9071cb77e8debdd98d8929c7b65401fc54 -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-musl-libgfortran5.tar.gz/md5/ff2b0ebdc7ef83cf8b48bd2ae76c6430 -CompilerSupportLibraries.v1.1.0+0.x86_64-linux-musl-libgfortran5.tar.gz/sha512/0730ecf1b9476612cadc3f3e7c1b227a1967edc091c88cd0cc19477079d1739fd5e7b1022ff686c0c6a2404edaebfb02c810dcfc1aa4187e7ecddb54998ad96c -CompilerSupportLibraries.v1.1.0+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/md5/35642304a9a2f435cf5214b2715198fe -CompilerSupportLibraries.v1.1.0+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/sha512/a67f41ba31c99a064f504f508711537f9e90089ca5352bfc2698c3fcd3e499ca716f07ffeac4fb1b88c2c934f7f380f262af8c863d3b16ac7e805d5c805ab358 -CompilerSupportLibraries.v1.1.0+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/md5/01df0fbb265e5ff1a480a7a5e23b0835 -CompilerSupportLibraries.v1.1.0+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/sha512/57a79f2b8e846c1514dcb18420f26ae2889962040f410b746836cab4395749155fa9cd9d00d4c25954c0ffa72f9f3823b1b50688a20ddf675301f64e0d4b5c7e -CompilerSupportLibraries.v1.1.0+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/1f1f6380ce8815cc9cedcea0b40860e7 -CompilerSupportLibraries.v1.1.0+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/a88ea8af8c8df792861812bfdf7f1bcaae31582ab78ce78b47a0dc6fd57b93441c0471f529ce23877131ac9701c6eed72ce89241746e18271f3686fbd718138c -CompilerSupportLibraries.v1.1.0+0.x86_64-w64-mingw32-libgfortran3.tar.gz/md5/5eab740e86bfa7656f6a08038fe2fa63 -CompilerSupportLibraries.v1.1.0+0.x86_64-w64-mingw32-libgfortran3.tar.gz/sha512/3dc6b7ec39ff7dcb71478376c86ce34a35a62f049f6203722c5414b7b635ff1b412e02d8d24c13c123d18b2e914780da4639538676694e342a1a6b507691ef25 -CompilerSupportLibraries.v1.1.0+0.x86_64-w64-mingw32-libgfortran4.tar.gz/md5/9718c79244ed31c367e715f1f563b8cd -CompilerSupportLibraries.v1.1.0+0.x86_64-w64-mingw32-libgfortran4.tar.gz/sha512/eec2380c4e182f4e923142736a2c4aaf11a525a5f966fed7e4ec4b431ee28f3842a4e73495df116604f74b419e6d398576ee3dd21d3c0c53b92167dcfd0f6b84 -CompilerSupportLibraries.v1.1.0+0.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/58d7b0b79a22f3aade7e4f39eec898e7 -CompilerSupportLibraries.v1.1.0+0.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/efecc0ca09ec6b7b8898c2ffd333c7e0a6a44706d72ac0e5010409aba92ee70a88b6fd77434bedafe0e013561f8d0c74b5a274808a6c9499f6a3005a7691785f +CompilerSupportLibraries.v1.1.1+0.aarch64-apple-darwin-libgfortran5.tar.gz/md5/20ebaad57850393b6ac9fa924e511fe4 +CompilerSupportLibraries.v1.1.1+0.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/020de4d8b0ff6bedbadaa305ff8445e6849f12053762ea4aa68412d1ec763dbd86f479587a2fbb862487f1feb04d976c38099ddf3887817a3d32b3f029cf85b1 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-gnu-libgfortran3.tar.gz/md5/d641904255ee412c45b089d92c53262b +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-gnu-libgfortran3.tar.gz/sha512/ace0383fe9bd64faeed1fb05a11bbec932bd56b8460d06d2b7c3e1b5f4f6e9a9b3345937088684e5cd1ca9a85ef1a5ff56a97a1f60449cd6e35247de1e123d81 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-gnu-libgfortran4.tar.gz/md5/2a71f320d8b9242ad26aabed74cbf404 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-gnu-libgfortran4.tar.gz/sha512/03e2a4482baaca2d6ce5cc207224d03bd7851486ebe8072c7317f5fcdd641395d945552d9462ab44a9f2e4b0ffaa3874a76f314d67bc0f75393a1151ab518611 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-gnu-libgfortran5.tar.gz/md5/1beec15ad689a5f572040ca2a7b6a880 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/27bbe212a8d43e841cf8f3e9964b72bc220fea03cf5e65721b02d2f3aa5193acdce41e512578ed6be935b413cd0d2224a6bcd2e9624931f39092ba3cfc5cbcc0 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-musl-libgfortran3.tar.gz/md5/9e949c2efe48a7b2a62bff7e1ffdede0 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-musl-libgfortran3.tar.gz/sha512/2947acb250f8ff4936da5ed02ddbfa492fc38bc87baa588a36bb892ba68b6636a912cda976f8fff00cc7a710c3bfb185826b4cd4a726750ef5f161d5f1aa21a2 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-musl-libgfortran4.tar.gz/md5/7202764b1a89a748b07460d9c40a9279 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-musl-libgfortran4.tar.gz/sha512/63236225a9becdd166c4395ea5081c64f57bc51af89c2edb5abeb419d6eb8224a380a633afd861bb84a12435fd19c8554cbe5ffadf8324ff2c7f17021ed53e69 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-musl-libgfortran5.tar.gz/md5/f66c30d3cec8057ae47f05df022ead51 +CompilerSupportLibraries.v1.1.1+0.aarch64-linux-musl-libgfortran5.tar.gz/sha512/5329d9469bb0f47560e52b15eb21ab70e0e2da0275bdb2f8e6ed4feb132bc9989a6b44984329455104546c95d05a05f8fb4f1cf232856219ba005100f4b16dc3 +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/md5/05ff63780f5b7c8c6c590c3626f32ac0 +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/sha512/8d3c4149531f3782f5efbb6a6fbbb7080ba005298ba962b5bc5f66250ea9fde91b34836ed909c16f306d21d2e358f985360962e9362a8e807ccd4254da3bb19b +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/md5/3ca2b6e8101d831e546c1b6ed2ca9a42 +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/sha512/21a0b9c5acde96c0a91303f4f395e55f272d5585ad18f0365105188d129a3ca94ad66d4dd99b471abdf41a7a7262a3b258fd04b887110ad15255b284cd1612b0 +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/d4d560b8ecce0ff2cb4dbc88cb25942a +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/d405f61525af1b2fe85107a70ed67b8a1eb767923487fa71539e0f49d6e70358c8a24f4ef1c224256cf677af99b54a2f8243f1e207350fcb14d426a7a6bb3915 +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/md5/8c6eddaa156fd0afee28ac5a154bc3f7 +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/sha512/b9fc86bb706ad98d61b63eb4cc8bfce6b2c67b58ba2cebecea7574f44790cce044bb1b4db1d20050b59538fa43b51cb352d752c77333a0f0621fde47c63a3596 +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/md5/0a54c16fea86c6dadb39eff65c465528 +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/sha512/c635c636384d3af5b4b078be7398fbc665a185eae69dd223279affb4836fb5c575d6ab296ae940ccbe73777bdb5e355f4f28a2fa27606ac143ff424641c60c65 +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/892dfd91703f0f77d170a5371a1c25d4 +CompilerSupportLibraries.v1.1.1+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/8ac59d00192c0e847168e61b3e93957f3909aab59ba8d05e47686a9f8b7226496f89b932151c42198ec966ccd47721cdf547a247ea4e5c61b22bfccce2ec591c +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/md5/05ff63780f5b7c8c6c590c3626f32ac0 +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/sha512/8d3c4149531f3782f5efbb6a6fbbb7080ba005298ba962b5bc5f66250ea9fde91b34836ed909c16f306d21d2e358f985360962e9362a8e807ccd4254da3bb19b +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/md5/3ca2b6e8101d831e546c1b6ed2ca9a42 +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/sha512/21a0b9c5acde96c0a91303f4f395e55f272d5585ad18f0365105188d129a3ca94ad66d4dd99b471abdf41a7a7262a3b258fd04b887110ad15255b284cd1612b0 +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/d4d560b8ecce0ff2cb4dbc88cb25942a +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/d405f61525af1b2fe85107a70ed67b8a1eb767923487fa71539e0f49d6e70358c8a24f4ef1c224256cf677af99b54a2f8243f1e207350fcb14d426a7a6bb3915 +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/md5/8c6eddaa156fd0afee28ac5a154bc3f7 +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/sha512/b9fc86bb706ad98d61b63eb4cc8bfce6b2c67b58ba2cebecea7574f44790cce044bb1b4db1d20050b59538fa43b51cb352d752c77333a0f0621fde47c63a3596 +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/md5/0a54c16fea86c6dadb39eff65c465528 +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/sha512/c635c636384d3af5b4b078be7398fbc665a185eae69dd223279affb4836fb5c575d6ab296ae940ccbe73777bdb5e355f4f28a2fa27606ac143ff424641c60c65 +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/892dfd91703f0f77d170a5371a1c25d4 +CompilerSupportLibraries.v1.1.1+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/8ac59d00192c0e847168e61b3e93957f3909aab59ba8d05e47686a9f8b7226496f89b932151c42198ec966ccd47721cdf547a247ea4e5c61b22bfccce2ec591c +CompilerSupportLibraries.v1.1.1+0.i686-linux-gnu-libgfortran3.tar.gz/md5/3094705222b6b61fd6a10422a73e1149 +CompilerSupportLibraries.v1.1.1+0.i686-linux-gnu-libgfortran3.tar.gz/sha512/27f874cde357ffa45aaa10f2e620ec0f8ab4e5a8bf4607fc023a2ec42040bcc9a724f959237c340d67451f8621402fa05133c1420086b87135f40326c30b97af +CompilerSupportLibraries.v1.1.1+0.i686-linux-gnu-libgfortran4.tar.gz/md5/ba0acaff60648efa3915348a8a353df8 +CompilerSupportLibraries.v1.1.1+0.i686-linux-gnu-libgfortran4.tar.gz/sha512/0b6aaf75363cbe6133ca3aed351ab58ef1e441f61375f5baf702d8043813c459d48e8af17630f1a07dc22772ec9b02076af33726ed94e6314ae37d5a139d6dcc +CompilerSupportLibraries.v1.1.1+0.i686-linux-gnu-libgfortran5.tar.gz/md5/95f1d57cfc43677e40bfc121bce79274 +CompilerSupportLibraries.v1.1.1+0.i686-linux-gnu-libgfortran5.tar.gz/sha512/edacd9960e9de1236c91752e103cddfc018d697e87fabb3cceadf36153b4e97842ef284bd1532290a5620007234882b4c4cd4f36525b61763d97b2f608358262 +CompilerSupportLibraries.v1.1.1+0.i686-linux-musl-libgfortran3.tar.gz/md5/f37fe1818e1634476c44afae478611c8 +CompilerSupportLibraries.v1.1.1+0.i686-linux-musl-libgfortran3.tar.gz/sha512/6e4e3eb5ac9570bfdf5280f59167eb6c4a74f3aa152afb4c5d180b9a6cdbdca557e7dd13f0b5b76943b45a65e848fe77c5b3bbc6ddb0fd846d03fbc9fbedf7ce +CompilerSupportLibraries.v1.1.1+0.i686-linux-musl-libgfortran4.tar.gz/md5/b4ffd52179aa0006c56f279b87cb7556 +CompilerSupportLibraries.v1.1.1+0.i686-linux-musl-libgfortran4.tar.gz/sha512/a047ac7db204c31802f646351af51c55fe06498e851b2df58d7f93f75d9c0067f8736f247f108991ec01ac7f86f3026ecf58b5f2f3a76d7eab00130754e7f704 +CompilerSupportLibraries.v1.1.1+0.i686-linux-musl-libgfortran5.tar.gz/md5/2d38fc835f236f89f457fdf859ccb903 +CompilerSupportLibraries.v1.1.1+0.i686-linux-musl-libgfortran5.tar.gz/sha512/51fbe41efbce33b1cf3728df6fa59fd0e85a13308b3e868fe9f70f4d67857615f83542ba69be824a73e89959503dd7a11335d1c495704bd7d6cad6656d0c5d57 +CompilerSupportLibraries.v1.1.1+0.i686-w64-mingw32-libgfortran3.tar.gz/md5/9650002f6729c0964d33afcab334d77d +CompilerSupportLibraries.v1.1.1+0.i686-w64-mingw32-libgfortran3.tar.gz/sha512/0b7907811a13d09b7b33203c7e46888308c7d6fcf5d69790babafc39f640541551f784264247f159a552f15df1ddd061c421a93b983d838d3bd7f85ba6427f70 +CompilerSupportLibraries.v1.1.1+0.i686-w64-mingw32-libgfortran4.tar.gz/md5/47e9fb99906b9647e26e4126a913074e +CompilerSupportLibraries.v1.1.1+0.i686-w64-mingw32-libgfortran4.tar.gz/sha512/d7285691fbe1318e48e061d678e54890762cc16996652a34b190924cc1462d24ab0b08729945eb25f4bef60e60d50f3e78db57d4cda0302b8ba579db8a1311e1 +CompilerSupportLibraries.v1.1.1+0.i686-w64-mingw32-libgfortran5.tar.gz/md5/b588b2710f2b83d2c70c6104e585a3bd +CompilerSupportLibraries.v1.1.1+0.i686-w64-mingw32-libgfortran5.tar.gz/sha512/b62a63b0c8750f85fc265db88456307b794e912352a68997c7cce06444391307c03edbe5b901833f53c5bd55f5a1e61a586538b08487cc139a2d71fccdce1d31 +CompilerSupportLibraries.v1.1.1+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/md5/7cce4f3dc057ebebaa677bf6f0d51e9e +CompilerSupportLibraries.v1.1.1+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/sha512/a0dd93905f0ede4da5e2fbacf2579154db8ac8e9963c77fb62284489686f2aa372925b3341742d86430a839267421af55f6e1e413473d17f13a1a199e6a904a0 +CompilerSupportLibraries.v1.1.1+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/md5/06ee6aaeca78b3e9005f53f1fa32731f +CompilerSupportLibraries.v1.1.1+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/sha512/ff0e33ce9f93b3a867cf409b95e763efbc8f4dde65ed19107eb14d29460d084f253e03ebd6375f1da996182b3d96e1fda4abff06507258da9a89ece36663db84 +CompilerSupportLibraries.v1.1.1+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/483251d28076ee959dff131d13d7e53b +CompilerSupportLibraries.v1.1.1+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/a7c9053a8c1b784cb6459762f26e0c2106a9758cbe2aefe8975a14aaaf61b8a08e51c465e733e44d01537beb59d467c57e536ebd8b27b7b68f46945174c469c7 +CompilerSupportLibraries.v1.1.1+0.x86_64-apple-darwin-libgfortran3.tar.gz/md5/a147bf3a6d6550c177b8a784b9b02e21 +CompilerSupportLibraries.v1.1.1+0.x86_64-apple-darwin-libgfortran3.tar.gz/sha512/c6f7a13f0195eae8f7ad980a4b24de9b155be69c4437522723411f9866a4aee3c5b350ee2f0c95f41f19aba43acaca78309881157e8498df0664c902d0c05a5d +CompilerSupportLibraries.v1.1.1+0.x86_64-apple-darwin-libgfortran4.tar.gz/md5/3f19c9d0e723a8d5591357ac3a9452a0 +CompilerSupportLibraries.v1.1.1+0.x86_64-apple-darwin-libgfortran4.tar.gz/sha512/5752bac310d80ed2dc1fc3d6580300d185787b9b933e31c8e0f572099abd0727d9483da8f9af858f706e96a183d2b10702c44381a080438cbb17d6459321ccfb +CompilerSupportLibraries.v1.1.1+0.x86_64-apple-darwin-libgfortran5.tar.gz/md5/ad0f0e2fe3e7d147a0a27271a2aba0fc +CompilerSupportLibraries.v1.1.1+0.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/f42231adea3d0b6133c3b5bc5fbf765bc6a7ba8ef0f407fa1b8def36dd8a71d20ef39fb6e57b43208489c2795a96562cdbf15f3d20b3f3a09edb29b99d19a33a +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-gnu-libgfortran3.tar.gz/md5/4c78d56dbbbff682c0a78d11fb9d1e70 +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-gnu-libgfortran3.tar.gz/sha512/0e9d6dcc4b8fddaaa94a26a46e915d33fb474f8a8ee14edd4d1c7e774846c44c5c5d852649a4f70409c99ac0e1d458077b7f0eb7dc0b0326ee8b625644d7074d +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-gnu-libgfortran4.tar.gz/md5/039d37f813b183c75feebadd21011eb6 +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-gnu-libgfortran4.tar.gz/sha512/05e7291de1fd2520247402f0db9d348fdd7a02d8dd9133ac65701f88d237110a3cc6c6e2c5717364ab786b6e6063038ec10c9605e77bc4dbe1064a0e77617f5d +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-gnu-libgfortran5.tar.gz/md5/a985f13a85eb14d1b6339ba4983dc372 +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/27468ccd5642e6e11bd5972684518a0fb883bf4835ac18f5279c3fce97b1779131c7d9e39d8de26a15c293c832946334e964919f51d7679cd0569ce82b938579 +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-musl-libgfortran3.tar.gz/md5/9d86ce2fe481ea97a1fd098bd47d524c +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-musl-libgfortran3.tar.gz/sha512/a865a4127bacaedd81b6c81279f6a44bc3497ab29a0401f66da1abfc0738ea459be9f158d06969c161a65925739665084bec5f8650a8cd1e8f0d08f1f44d729f +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-musl-libgfortran4.tar.gz/md5/86d9db869a7af6c96dea39f5d9d90505 +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-musl-libgfortran4.tar.gz/sha512/01e0c69b04138989200ded92eddae6ff1873d3a440d17273d08bee40d53b2929e35bfd14be051074fe78671cac34ac2dd7360c1571790ee52f94a5921de42a65 +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-musl-libgfortran5.tar.gz/md5/e72d28df4bcb60ab2f3389046e7c83a8 +CompilerSupportLibraries.v1.1.1+0.x86_64-linux-musl-libgfortran5.tar.gz/sha512/cac193a26328ddeff5f7bcc3d7207101c574f9bdb1bff5c2b925315c5c2404a2fdb6591d1968f30931373fbfcae9bda784c72e65580ad3acc398448cd193f65d +CompilerSupportLibraries.v1.1.1+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/md5/35642304a9a2f435cf5214b2715198fe +CompilerSupportLibraries.v1.1.1+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/sha512/a67f41ba31c99a064f504f508711537f9e90089ca5352bfc2698c3fcd3e499ca716f07ffeac4fb1b88c2c934f7f380f262af8c863d3b16ac7e805d5c805ab358 +CompilerSupportLibraries.v1.1.1+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/md5/01df0fbb265e5ff1a480a7a5e23b0835 +CompilerSupportLibraries.v1.1.1+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/sha512/57a79f2b8e846c1514dcb18420f26ae2889962040f410b746836cab4395749155fa9cd9d00d4c25954c0ffa72f9f3823b1b50688a20ddf675301f64e0d4b5c7e +CompilerSupportLibraries.v1.1.1+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/1f1f6380ce8815cc9cedcea0b40860e7 +CompilerSupportLibraries.v1.1.1+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/a88ea8af8c8df792861812bfdf7f1bcaae31582ab78ce78b47a0dc6fd57b93441c0471f529ce23877131ac9701c6eed72ce89241746e18271f3686fbd718138c +CompilerSupportLibraries.v1.1.1+0.x86_64-w64-mingw32-libgfortran3.tar.gz/md5/38fc8c445a1a610db40a7609155e22d6 +CompilerSupportLibraries.v1.1.1+0.x86_64-w64-mingw32-libgfortran3.tar.gz/sha512/085652c7ca583c3623611ca9262b70765c9936c9feb5f9034b2c6b6d6677a7a1d7d201b83d82d0d268f3190bd1a62eab0124e8fae3625407dee7f1df89d4106c +CompilerSupportLibraries.v1.1.1+0.x86_64-w64-mingw32-libgfortran4.tar.gz/md5/f3f89eb3c2e441fde6e6b9c1c1a61183 +CompilerSupportLibraries.v1.1.1+0.x86_64-w64-mingw32-libgfortran4.tar.gz/sha512/c53f79e20ad043ab099873f38ece98c6bed22950610ba88b9c178a4bd943039cc426473828d509deb8c65c93309da1de87bdf36fb3954b8f8047277c418fe2e0 +CompilerSupportLibraries.v1.1.1+0.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/024f7133425db23e215dc55589bb9171 +CompilerSupportLibraries.v1.1.1+0.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/819945496ea48dd44d8c0f12a11a358b7d1ebf198d60fbad576d74ddee68cdea98070cdd11ca96567d0c772ec007c03cbc83ff5c7d2ad737cbd486fe0c9afcd5 diff --git a/stdlib/CompilerSupportLibraries_jll/Project.toml b/stdlib/CompilerSupportLibraries_jll/Project.toml index 3e15ff6b87b71..5aab865b5f6fc 100644 --- a/stdlib/CompilerSupportLibraries_jll/Project.toml +++ b/stdlib/CompilerSupportLibraries_jll/Project.toml @@ -4,7 +4,7 @@ uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" # NOTE: When updating this, also make sure to update the value # `CSL_NEXT_GLIBCXX_VERSION` in `Make.inc`, to properly disable # automatic usage of BB-built CSLs on extremely up-to-date systems! -version = "1.1.0+0" +version = "1.1.1+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" From ab37ee59bac60ce7162566f31f85d5fa6208fa8d Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 10 Mar 2024 18:34:55 -0400 Subject: [PATCH 40/55] move precompile workload back from Base (#53679) (cherry picked from commit 26b3b5fb18d3b03e2e79be9e3bf817afeb55b711) --- base/Base.jl | 55 ---------------------- contrib/generate_precompile.jl | 84 +++++++++++++++++++++------------- 2 files changed, 52 insertions(+), 87 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index c6a1113996fc3..491725b29f853 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -567,61 +567,6 @@ end_base_include = time_ns() const _sysimage_modules = PkgId[] in_sysimage(pkgid::PkgId) = pkgid in _sysimage_modules -# Precompiles for Revise and other packages -# TODO: move these to contrib/generate_precompile.jl -# The problem is they don't work there -for match = _methods(+, (Int, Int), -1, get_world_counter()) - m = match.method - delete!(push!(Set{Method}(), m), m) - copy(Core.Compiler.retrieve_code_info(Core.Compiler.specialize_method(match), typemax(UInt))) - - empty!(Set()) - push!(push!(Set{Union{GlobalRef,Symbol}}(), :two), GlobalRef(Base, :two)) - (setindex!(Dict{String,Base.PkgId}(), Base.PkgId(Base), "file.jl"))["file.jl"] - (setindex!(Dict{Symbol,Vector{Int}}(), [1], :two))[:two] - (setindex!(Dict{Base.PkgId,String}(), "file.jl", Base.PkgId(Base)))[Base.PkgId(Base)] - (setindex!(Dict{Union{GlobalRef,Symbol}, Vector{Int}}(), [1], :two))[:two] - (setindex!(IdDict{Type, Union{Missing, Vector{Tuple{LineNumberNode, Expr}}}}(), missing, Int))[Int] - Dict{Symbol, Union{Nothing, Bool, Symbol}}(:one => false)[:one] - Dict(Base => [:(1+1)])[Base] - Dict(:one => [1])[:one] - Dict("abc" => Set())["abc"] - pushfirst!([], sum) - get(Base.pkgorigins, Base.PkgId(Base), nothing) - sort!([1,2,3]) - unique!([1,2,3]) - cumsum([1,2,3]) - append!(Int[], BitSet()) - isempty(BitSet()) - delete!(BitSet([1,2]), 3) - deleteat!(Int32[1,2,3], [1,3]) - deleteat!(Any[1,2,3], [1,3]) - Core.svec(1, 2) == Core.svec(3, 4) - any(t->t[1].line > 1, [(LineNumberNode(2,:none), :(1+1))]) - - # Code loading uses this - sortperm(mtime.(readdir(".")), rev=true) - # JLLWrappers uses these - Dict{UUID,Set{String}}()[UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210")] = Set{String}() - get!(Set{String}, Dict{UUID,Set{String}}(), UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210")) - eachindex(IndexLinear(), Expr[]) - push!(Expr[], Expr(:return, false)) - vcat(String[], String[]) - k, v = (:hello => nothing) - precompile(indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int)) - precompile(indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int)) - # Preferences uses these - precompile(get_preferences, (UUID,)) - precompile(record_compiletime_preference, (UUID, String)) - get(Dict{String,Any}(), "missing", nothing) - delete!(Dict{String,Any}(), "missing") - for (k, v) in Dict{String,Any}() - println(k) - end - - break # only actually need to do this once -end - if is_primary_base_module # Profiling helper diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 57af6c1f32679..9e4e2d4485189 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -91,39 +91,59 @@ for T in (Float16, Float32, Float64), IO in (IOBuffer, IOContext{IOBuffer}, Base hardcoded_precompile_statements *= "precompile(Tuple{typeof(show), $IO, $T})\n" end +# Precompiles for Revise and other packages precompile_script = """ -# NOTE: these were moved to the end of Base.jl. TODO: move back here. -# # Used by Revise & its dependencies -# while true # force inference -# delete!(push!(Set{Module}(), Base), Main) -# m = first(methods(+)) -# delete!(push!(Set{Method}(), m), m) -# empty!(Set()) -# push!(push!(Set{Union{GlobalRef,Symbol}}(), :two), GlobalRef(Base, :two)) -# (setindex!(Dict{String,Base.PkgId}(), Base.PkgId(Base), "file.jl"))["file.jl"] -# (setindex!(Dict{Symbol,Vector{Int}}(), [1], :two))[:two] -# (setindex!(Dict{Base.PkgId,String}(), "file.jl", Base.PkgId(Base)))[Base.PkgId(Base)] -# (setindex!(Dict{Union{GlobalRef,Symbol}, Vector{Int}}(), [1], :two))[:two] -# (setindex!(IdDict{Type, Union{Missing, Vector{Tuple{LineNumberNode, Expr}}}}(), missing, Int))[Int] -# Dict{Symbol, Union{Nothing, Bool, Symbol}}(:one => false)[:one] -# Dict(Base => [:(1+1)])[Base] -# Dict(:one => [1])[:one] -# Dict("abc" => Set())["abc"] -# pushfirst!([], sum) -# get(Base.pkgorigins, Base.PkgId(Base), nothing) -# sort!([1,2,3]) -# unique!([1,2,3]) -# cumsum([1,2,3]) -# append!(Int[], BitSet()) -# isempty(BitSet()) -# delete!(BitSet([1,2]), 3) -# deleteat!(Int32[1,2,3], [1,3]) -# deleteat!(Any[1,2,3], [1,3]) -# Core.svec(1, 2) == Core.svec(3, 4) -# # copy(Core.Compiler.retrieve_code_info(Core.Compiler.specialize_method(which(+, (Int, Int)), [Int, Int], Core.svec()))) -# any(t->t[1].line > 1, [(LineNumberNode(2,:none),:(1+1))]) -# break # end force inference -# end +for match = Base._methods(+, (Int, Int), -1, Base.get_world_counter()) + m = match.method + delete!(push!(Set{Method}(), m), m) + copy(Core.Compiler.retrieve_code_info(Core.Compiler.specialize_method(match), typemax(UInt))) + + empty!(Set()) + push!(push!(Set{Union{GlobalRef,Symbol}}(), :two), GlobalRef(Base, :two)) + (setindex!(Dict{String,Base.PkgId}(), Base.PkgId(Base), "file.jl"))["file.jl"] + (setindex!(Dict{Symbol,Vector{Int}}(), [1], :two))[:two] + (setindex!(Dict{Base.PkgId,String}(), "file.jl", Base.PkgId(Base)))[Base.PkgId(Base)] + (setindex!(Dict{Union{GlobalRef,Symbol}, Vector{Int}}(), [1], :two))[:two] + (setindex!(IdDict{Type, Union{Missing, Vector{Tuple{LineNumberNode, Expr}}}}(), missing, Int))[Int] + Dict{Symbol, Union{Nothing, Bool, Symbol}}(:one => false)[:one] + Dict(Base => [:(1+1)])[Base] + Dict(:one => [1])[:one] + Dict("abc" => Set())["abc"] + pushfirst!([], sum) + get(Base.pkgorigins, Base.PkgId(Base), nothing) + sort!([1,2,3]) + unique!([1,2,3]) + cumsum([1,2,3]) + append!(Int[], BitSet()) + isempty(BitSet()) + delete!(BitSet([1,2]), 3) + deleteat!(Int32[1,2,3], [1,3]) + deleteat!(Any[1,2,3], [1,3]) + Core.svec(1, 2) == Core.svec(3, 4) + any(t->t[1].line > 1, [(LineNumberNode(2,:none), :(1+1))]) + + # Code loading uses this + sortperm(mtime.(readdir(".")), rev=true) + # JLLWrappers uses these + Dict{Base.UUID,Set{String}}()[Base.UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210")] = Set{String}() + get!(Set{String}, Dict{Base.UUID,Set{String}}(), Base.UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210")) + eachindex(IndexLinear(), Expr[]) + push!(Expr[], Expr(:return, false)) + vcat(String[], String[]) + k, v = (:hello => nothing) + precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int)) + precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int)) + # Preferences uses these + precompile(Base.get_preferences, (Base.UUID,)) + precompile(Base.record_compiletime_preference, (Base.UUID, String)) + get(Dict{String,Any}(), "missing", nothing) + delete!(Dict{String,Any}(), "missing") + for (k, v) in Dict{String,Any}() + println(k) + end + + break # only actually need to do this once +end """ julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename()) From 15f13f7c575d159d088ab73163e7b54bb226aa7d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 11 Mar 2024 08:28:24 -0400 Subject: [PATCH 41/55] add isassigned methods for reinterpretarray (#53663) Fixes #52925 Refs #51760 (cherry picked from commit 60d4b7be63a477071a0640a3bc87d5db3ff73d93) --- base/reinterpretarray.jl | 4 ++++ test/reinterpretarray.jl | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index ff674690a5c66..8b4025e6903cd 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -386,6 +386,10 @@ check_ptr_indexable(a::AbstractArray, sz) = false @propagate_inbounds getindex(a::ReinterpretArray) = a[firstindex(a)] +@propagate_inbounds isassigned(a::ReinterpretArray, inds::Integer...) = checkbounds(Bool, a, inds...) && (check_ptr_indexable(a) || _isassigned_ra(a, inds...)) +@propagate_inbounds isassigned(a::ReinterpretArray, inds::SCartesianIndex2) = isassigned(a.parent, inds.j) +@propagate_inbounds _isassigned_ra(a::ReinterpretArray, inds...) = true # that is not entirely true, but computing exactly which indexes will be accessed in the parent requires a lot of duplication from the _getindex_ra code + @propagate_inbounds function getindex(a::ReinterpretArray{T,N,S}, inds::Vararg{Int, N}) where {T,N,S} check_readable(a) check_ptr_indexable(a) && return _getindex_ptr(a, inds...) diff --git a/test/reinterpretarray.jl b/test/reinterpretarray.jl index 05a40895b7934..46ecbf6d06723 100644 --- a/test/reinterpretarray.jl +++ b/test/reinterpretarray.jl @@ -588,3 +588,23 @@ end @test_throws ArgumentError reinterpret(Tuple{Int32, Int64}, (Int16(1), Int64(4))) end + +let R = reinterpret(Float32, ComplexF32[1.0f0+2.0f0*im, 4.0f0+3.0f0*im]) + @test !isassigned(R, 0) + @test isassigned(R, 1) + @test isassigned(R, 4) + @test isassigned(R, Int8(2), Int16(1), Int32(1), Int64(1)) + @test !isassigned(R, 1, 2) + @test !isassigned(R, 5) + @test Array(R)::Vector{Float32} == [1.0f0, 2.0f0, 4.0f0, 3.0f0] +end + +let R = reinterpret(reshape, Float32, ComplexF32[1.0f0+2.0f0*im, 4.0f0+3.0f0*im]) + @test !isassigned(R, 0) + @test isassigned(R, 1) + @test isassigned(R, 4) + @test isassigned(R, Int8(2), Int16(2), Int32(1), Int64(1)) + @test !isassigned(R, 1, 1, 2) + @test !isassigned(R, 5) + @test Array(R)::Matrix{Float32} == [1.0f0 4.0f0; 2.0f0 3.0f0] +end From 21a26025174d18035939aa905d51dd93382991a0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 11 Mar 2024 08:28:49 -0400 Subject: [PATCH 42/55] [REPL] fix incorrectly cleared line after completions accepted (#53662) The hint must be cleared before the screen state is reset, otherwise the state after reset may not be compatible with being able to clear it. Fixes #52264 (cherry picked from commit 2978a642b8d0a1912f80e84e7c2e769919aed376) --- stdlib/REPL/src/LineEdit.jl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index 9f1dd76b168af..4c95d4264dbcb 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -480,14 +480,12 @@ prompt_string(f::Function) = Base.invokelatest(f) function maybe_show_hint(s::PromptState) isa(s.hint, String) || return nothing # The hint being "" then nothing is used to first clear a previous hint, then skip printing the hint - # the clear line cannot be printed each time because it breaks column movement if isempty(s.hint) - print(terminal(s), "\e[0K") # clear remainder of line which had a hint s.hint = nothing else Base.printstyled(terminal(s), s.hint, color=:light_black) cmove_left(terminal(s), textwidth(s.hint)) - s.hint = "" # being "" signals to do one clear line remainder to clear the hint next time if still empty + s.hint = "" # being "" signals to do one clear line remainder to clear the hint next time the screen is refreshed end return nothing end @@ -497,8 +495,13 @@ function refresh_multi_line(s::PromptState; kw...) close(s.refresh_wait) s.refresh_wait = nothing end + if s.hint isa String + # clear remainder of line which is unknown here if it had a hint before unbeknownst to refresh_multi_line + # the clear line cannot be printed each time because it would break column movement + print(terminal(s), "\e[0K") + end r = refresh_multi_line(terminal(s), s; kw...) - maybe_show_hint(s) + maybe_show_hint(s) # now maybe write the hint back to the screen return r end refresh_multi_line(s::ModeState; kw...) = refresh_multi_line(terminal(s), s; kw...) From fd9c49c5c8c299892cf6488b72aaee00a0455ad3 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Mon, 11 Mar 2024 19:45:30 +0530 Subject: [PATCH 43/55] Linalg: matprod_dest for Diagonal and adjvec (#53611) (cherry picked from commit f882c00bdc5904cee7d1631740ba3824157bc656) --- stdlib/LinearAlgebra/src/LinearAlgebra.jl | 2 ++ stdlib/LinearAlgebra/test/diagonal.jl | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/stdlib/LinearAlgebra/src/LinearAlgebra.jl b/stdlib/LinearAlgebra/src/LinearAlgebra.jl index acacf4b20cfc7..cfa99525d2dcb 100644 --- a/stdlib/LinearAlgebra/src/LinearAlgebra.jl +++ b/stdlib/LinearAlgebra/src/LinearAlgebra.jl @@ -581,6 +581,8 @@ matprod_dest(A::Diagonal, B::StructuredMatrix, TS) = similar(B, TS) matprod_dest(A::Diagonal, B::Diagonal, TS) = similar(B, TS) matprod_dest(A::HermOrSym, B::Diagonal, TS) = similar(A, TS, size(A)) matprod_dest(A::Diagonal, B::HermOrSym, TS) = similar(B, TS, size(B)) +# Special handling for adj/trans vec +matprod_dest(A::Diagonal, B::AdjOrTransAbsVec, TS) = similar(B, TS) # TODO: remove once not used anymore in SparseArrays.jl # some trait like this would be cool diff --git a/stdlib/LinearAlgebra/test/diagonal.jl b/stdlib/LinearAlgebra/test/diagonal.jl index 18abc6940eded..3087e87f63415 100644 --- a/stdlib/LinearAlgebra/test/diagonal.jl +++ b/stdlib/LinearAlgebra/test/diagonal.jl @@ -1277,4 +1277,15 @@ end @test c == Diagonal([2,2,2,2]) end +@testset "mul/div with an adjoint vector" begin + A = [1.0;;] + x = [1.0] + yadj = Diagonal(A) \ x' + @test typeof(yadj) == typeof(x') + @test yadj == x' + yadj = Diagonal(A) * x' + @test typeof(yadj) == typeof(x') + @test yadj == x' +end + end # module TestDiagonal From 641ae2c2303bfd8484132912f743ea3679275fa5 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 11 Mar 2024 15:09:12 -0400 Subject: [PATCH 44/55] fix #52025, re-allow all implicit pointer casts in cconvert for Array (#53659) fix #52025 (cherry picked from commit f9e08f7941efad238c13641cb72a0d383fd7eaba) --- base/refpointer.jl | 4 ++-- test/ccall.jl | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/base/refpointer.jl b/base/refpointer.jl index ce2c6fc00560d..5027462eeb6b6 100644 --- a/base/refpointer.jl +++ b/base/refpointer.jl @@ -171,8 +171,8 @@ if is_primary_base_module Ref(x::AbstractArray, i::Integer) = RefArray(x, i) end -cconvert(::Type{Ptr{P}}, a::Array{P}) where {P<:Union{Ptr,Cwstring,Cstring}} = getfield(a, :ref) -cconvert(::Type{Ref{P}}, a::Array{P}) where {P<:Union{Ptr,Cwstring,Cstring}} = getfield(a, :ref) +cconvert(::Type{Ptr{P}}, a::Array{<:Union{Ptr,Cwstring,Cstring}}) where {P<:Union{Ptr,Cwstring,Cstring}} = getfield(a, :ref) +cconvert(::Type{Ref{P}}, a::Array{<:Union{Ptr,Cwstring,Cstring}}) where {P<:Union{Ptr,Cwstring,Cstring}} = getfield(a, :ref) cconvert(::Type{Ptr{P}}, a::Array) where {P<:Union{Ptr,Cwstring,Cstring}} = Ref{P}(a) cconvert(::Type{Ref{P}}, a::Array) where {P<:Union{Ptr,Cwstring,Cstring}} = Ref{P}(a) diff --git a/test/ccall.jl b/test/ccall.jl index 3647173eb9290..8b1fefdfc66e4 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -1934,3 +1934,6 @@ end end @test_throws "could not load symbol \"test\"" somefunction_not_found_libc() end + +# issue #52025 +@test Base.unsafe_convert(Ptr{Ptr{Cchar}}, Base.cconvert(Ptr{Ptr{Cchar}}, map(pointer, ["ab"]))) isa Ptr{Ptr{Cchar}} From 0087a1177509a6e70ff122636b0f7b6c596ce102 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Tue, 12 Mar 2024 07:34:25 +0530 Subject: [PATCH 45/55] LAPACK: validate input parameters to throw informative errors (#53631) This PR validates the input parameters to the Julia LAPACK wrappers, so that the error messages are more informative. On nightly ```julia julia> using LinearAlgebra julia> LAPACK.geev!('X', 'X', rand(2,2)) ** On entry to DGEEV parameter number 1 had an illegal value ERROR: ArgumentError: invalid argument #1 to LAPACK call ``` This PR ```julia julia> using LinearAlgebra julia> LAPACK.geev!('X', 'X', rand(2,2)) ERROR: ArgumentError: argument #1: jobvl must be one of ('N', 'V'), but 'X' was passed ``` Secondly, moved certain allocations (e.g. in `geevx`) below the validation checks, so that these only happen for valid parameter values. Thirdly, added `require_one_based_indexing` checks to functions where these were missing. (cherry picked from commit dcd1fb2148dd7affee9449e304a9b42e7feba1ce) --- stdlib/LinearAlgebra/src/blas.jl | 2 +- stdlib/LinearAlgebra/src/lapack.jl | 240 ++++++++++++++++++++++------ stdlib/LinearAlgebra/test/lapack.jl | 184 ++++++++++++++++----- 3 files changed, 344 insertions(+), 82 deletions(-) diff --git a/stdlib/LinearAlgebra/src/blas.jl b/stdlib/LinearAlgebra/src/blas.jl index 5d8821cd6e009..1245207171321 100644 --- a/stdlib/LinearAlgebra/src/blas.jl +++ b/stdlib/LinearAlgebra/src/blas.jl @@ -169,7 +169,7 @@ end "Check that upper/lower (for special matrices) is correctly specified" function chkuplo(uplo::AbstractChar) if !(uplo == 'U' || uplo == 'L') - throw(ArgumentError(lazy"uplo argument must be 'U' (upper) or 'L' (lower), got $uplo")) + throw(ArgumentError(lazy"uplo argument must be 'U' (upper) or 'L' (lower), got '$uplo'")) end uplo end diff --git a/stdlib/LinearAlgebra/src/lapack.jl b/stdlib/LinearAlgebra/src/lapack.jl index a85978435bf46..02dfa0079038b 100644 --- a/stdlib/LinearAlgebra/src/lapack.jl +++ b/stdlib/LinearAlgebra/src/lapack.jl @@ -55,10 +55,23 @@ function chkposdef(ret::BlasInt) end end +# Generic fallback function to assert that parameters are valid +# In specific cases, the following functions may be more useful +macro chkvalidparam(position::Int, param, validvalues) + :(chkvalidparam($position, $(string(param)), $(esc(param)), $validvalues)) +end +function chkvalidparam(position::Int, var::String, val, validvals) + if val ∉ validvals + throw(ArgumentError( + "argument #$position: $var must be one of $validvals, but $(repr(val)) was passed")) + end + return val +end + "Check that {c}transpose is correctly specified" function chktrans(trans::AbstractChar) if !(trans == 'N' || trans == 'C' || trans == 'T') - throw(ArgumentError("trans argument must be 'N' (no transpose), 'T' (transpose), or 'C' (conjugate transpose), got $trans")) + throw(ArgumentError("trans argument must be 'N' (no transpose), 'T' (transpose), or 'C' (conjugate transpose), got '$trans'")) end trans end @@ -66,7 +79,7 @@ end "Check that left/right hand side multiply is correctly specified" function chkside(side::AbstractChar) if !(side == 'L' || side == 'R') - throw(ArgumentError("side argument must be 'L' (left hand multiply) or 'R' (right hand multiply), got $side")) + throw(ArgumentError("side argument must be 'L' (left hand multiply) or 'R' (right hand multiply), got '$side'")) end side end @@ -74,7 +87,7 @@ end "Check that unit diagonal flag is correctly specified" function chkdiag(diag::AbstractChar) if !(diag == 'U' || diag =='N') - throw(ArgumentError("diag argument must be 'U' (unit diagonal) or 'N' (non-unit diagonal), got $diag")) + throw(ArgumentError("diag argument must be 'U' (unit diagonal) or 'N' (non-unit diagonal), got '$diag'")) end diag end @@ -93,6 +106,7 @@ end function chkuplofinite(A::AbstractMatrix, uplo::AbstractChar) require_one_based_indexing(A) + chkuplo(uplo) m, n = size(A) if uplo == 'U' @inbounds for j in 1:n, i in 1:j @@ -213,7 +227,9 @@ for (gebal, gebak, elty, relty) in # .. Array Arguments .. # DOUBLE PRECISION A( LDA, * ), SCALE( * ) function gebal!(job::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) chkstride1(A) + @chkvalidparam 1 job ('N', 'P', 'S', 'B') n = checksquare(A) chkfinite(A) # balancing routines don't support NaNs and Infs ihi = Ref{BlasInt}() @@ -238,6 +254,7 @@ for (gebal, gebak, elty, relty) in ilo::BlasInt, ihi::BlasInt, scale::AbstractVector{$relty}, V::AbstractMatrix{$elty}) require_one_based_indexing(scale, V) + @chkvalidparam 1 job ('N', 'P', 'S', 'B') chkstride1(scale, V) chkside(side) chkfinite(V) # balancing routines don't support NaNs and Infs @@ -624,11 +641,13 @@ does not equal zero then the `j`th column of `A` is permuted to the front of geqp3!(A::AbstractMatrix, jpvt::AbstractVector{BlasInt}, tau::AbstractVector) function geqp3!(A::AbstractMatrix{<:BlasFloat}, jpvt::AbstractVector{BlasInt}) + require_one_based_indexing(A, jpvt) m, n = size(A) geqp3!(A, jpvt, similar(A, min(m, n))) end function geqp3!(A::AbstractMatrix{<:BlasFloat}) + require_one_based_indexing(A) m, n = size(A) geqp3!(A, zeros(BlasInt, n), similar(A, min(m, n))) end @@ -777,6 +796,7 @@ for (larfg, elty) in # .. Array Arguments .. # DOUBLE PRECISION x( * ) function larfg!(x::AbstractVector{$elty}) + require_one_based_indexing(x) N = BlasInt(length(x)) α = Ref{$elty}(x[1]) incx = BlasInt(1) @@ -805,6 +825,7 @@ for (larf, elty) in # DOUBLE PRECISION c( ldc, * ), v( * ), work( * ) function larf!(side::AbstractChar, v::AbstractVector{$elty}, τ::$elty, C::AbstractMatrix{$elty}, work::AbstractVector{$elty}) + require_one_based_indexing(v, C, work) m, n = size(C) chkside(side) ldc = max(1, stride(C, 2)) @@ -820,6 +841,7 @@ for (larf, elty) in function larf!(side::AbstractChar, v::AbstractVector{$elty}, τ::$elty, C::AbstractMatrix{$elty}) + require_one_based_indexing(v, C) m, n = size(C) chkside(side) lwork = side == 'L' ? n : m @@ -1134,6 +1156,7 @@ for (gesvx, elty) in AF::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}, equed::AbstractChar, R::AbstractVector{$elty}, C::AbstractVector{$elty}, B::AbstractVecOrMat{$elty}) require_one_based_indexing(A, AF, ipiv, R, C, B) + @chkvalidparam 1 fact ('F', 'N', 'E') chktrans(trans) chkstride1(ipiv, R, C, B) n = checksquare(A) @@ -1168,6 +1191,7 @@ for (gesvx, elty) in end function gesvx!(A::AbstractMatrix{$elty}, B::AbstractVecOrMat{$elty}) + require_one_based_indexing(A, B) n = size(A,1) X, equed, R, C, B, rcond, ferr, berr, rpgf = gesvx!('N', 'N', A, @@ -1204,6 +1228,7 @@ for (gesvx, elty, relty) in AF::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}, equed::AbstractChar, R::AbstractVector{$relty}, C::AbstractVector{$relty}, B::AbstractVecOrMat{$elty}) require_one_based_indexing(A, AF, ipiv, R, C, B) + @chkvalidparam 1 fact ('F', 'N', 'E') chktrans(trans) chkstride1(A, AF, ipiv, R, C, B) n = checksquare(A) @@ -1239,6 +1264,7 @@ for (gesvx, elty, relty) in #Wrapper for the no-equilibration, no-transpose calculation function gesvx!(A::AbstractMatrix{$elty}, B::AbstractVecOrMat{$elty}) + require_one_based_indexing(A, B) n = size(A,1) X, equed, R, C, B, rcond, ferr, berr, rpgf = gesvx!('N', 'N', A, @@ -1577,8 +1603,11 @@ for (geev, gesvd, gesdd, ggsvd, elty, relty) in # DOUBLE PRECISION A( LDA, * ), VL( LDVL, * ), VR( LDVR, * ), # $ WI( * ), WORK( * ), WR( * ) function geev!(jobvl::AbstractChar, jobvr::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) + @chkvalidparam 1 jobvl ('N', 'V') + @chkvalidparam 2 jobvr ('N', 'V') chkfinite(A) # balancing routines don't support NaNs and Infs lvecs = jobvl == 'V' rvecs = jobvr == 'V' @@ -1635,6 +1664,7 @@ for (geev, gesvd, gesdd, ggsvd, elty, relty) in function gesdd!(job::AbstractChar, A::AbstractMatrix{$elty}) require_one_based_indexing(A) chkstride1(A) + @chkvalidparam 1 job ('A', 'S', 'O', 'N') m, n = size(A) minmn = min(m, n) if job == 'A' @@ -1716,6 +1746,9 @@ for (geev, gesvd, gesdd, ggsvd, elty, relty) in function gesvd!(jobu::AbstractChar, jobvt::AbstractChar, A::AbstractMatrix{$elty}) require_one_based_indexing(A) chkstride1(A) + @chkvalidparam 1 jobu ('A', 'S', 'O', 'N') + @chkvalidparam 2 jobvt ('A', 'S', 'O', 'N') + (jobu == jobvt == 'O') && throw(ArgumentError("jobu and jobvt cannot both be O")) m, n = size(A) minmn = min(m, n) S = similar(A, $relty, minmn) @@ -1785,6 +1818,9 @@ for (geev, gesvd, gesdd, ggsvd, elty, relty) in function ggsvd!(jobu::AbstractChar, jobv::AbstractChar, jobq::AbstractChar, A::AbstractMatrix{$elty}, B::AbstractMatrix{$elty}) require_one_based_indexing(A, B) chkstride1(A, B) + @chkvalidparam 1 jobu ('U', 'N') + @chkvalidparam 2 jobv ('V', 'N') + @chkvalidparam 3 jobq ('Q', 'N') m, n = size(A) if size(B, 2) != n throw(DimensionMismatch("B has second dimension $(size(B,2)) but needs $n")) @@ -1912,6 +1948,9 @@ for (f, elty) in ((:dggsvd3_, :Float64), function ggsvd3!(jobu::AbstractChar, jobv::AbstractChar, jobq::AbstractChar, A::AbstractMatrix{$elty}, B::AbstractMatrix{$elty}) require_one_based_indexing(A, B) chkstride1(A, B) + @chkvalidparam 1 jobu ('U', 'N') + @chkvalidparam 2 jobv ('V', 'N') + @chkvalidparam 3 jobq ('Q', 'N') m, n = size(A) if size(B, 2) != n throw(DimensionMismatch("B has second dimension $(size(B,2)) but needs $n")) @@ -1971,6 +2010,9 @@ for (f, elty, relty) in ((:zggsvd3_, :ComplexF64, :Float64), function ggsvd3!(jobu::AbstractChar, jobv::AbstractChar, jobq::AbstractChar, A::AbstractMatrix{$elty}, B::AbstractMatrix{$elty}) require_one_based_indexing(A, B) chkstride1(A, B) + @chkvalidparam 1 jobu ('U', 'N') + @chkvalidparam 2 jobv ('V', 'N') + @chkvalidparam 3 jobq ('Q', 'N') m, n = size(A) if size(B, 2) != n throw(DimensionMismatch("B has second dimension $(size(B,2)) but needs $n")) @@ -2057,14 +2099,13 @@ for (geevx, ggev, ggev3, elty) in # $ SCALE( * ), VL( LDVL, * ), VR( LDVR, * ), # $ WI( * ), WORK( * ), WR( * ) function geevx!(balanc::AbstractChar, jobvl::AbstractChar, jobvr::AbstractChar, sense::AbstractChar, A::AbstractMatrix{$elty}) - n = checksquare(A) - chkfinite(A) # balancing routines don't support NaNs and Infs - lda = max(1,stride(A,2)) - wr = similar(A, $elty, n) - wi = similar(A, $elty, n) - if balanc ∉ ['N', 'P', 'S', 'B'] - throw(ArgumentError("balanc must be 'N', 'P', 'S', or 'B', but $balanc was passed")) + require_one_based_indexing(A) + @chkvalidparam 1 balanc ('N', 'P', 'S', 'B') + @chkvalidparam 4 sense ('N', 'E', 'V', 'B') + if sense ∈ ('E', 'B') && !(jobvl == jobvr == 'V') + throw(ArgumentError("sense = '$sense' requires jobvl = 'V' and jobvr = 'V'")) end + n = checksquare(A) ldvl = 0 if jobvl == 'V' ldvl = n @@ -2073,7 +2114,6 @@ for (geevx, ggev, ggev3, elty) in else throw(ArgumentError("jobvl must be 'V' or 'N', but $jobvl was passed")) end - VL = similar(A, $elty, ldvl, n) ldvr = 0 if jobvr == 'V' ldvr = n @@ -2082,6 +2122,11 @@ for (geevx, ggev, ggev3, elty) in else throw(ArgumentError("jobvr must be 'V' or 'N', but $jobvr was passed")) end + chkfinite(A) # balancing routines don't support NaNs and Infs + lda = max(1,stride(A,2)) + wr = similar(A, $elty, n) + wi = similar(A, $elty, n) + VL = similar(A, $elty, ldvl, n) VR = similar(A, $elty, ldvr, n) ilo = Ref{BlasInt}() ihi = Ref{BlasInt}() @@ -2143,11 +2188,6 @@ for (geevx, ggev, ggev3, elty) in if n != m throw(DimensionMismatch("A has dimensions $(size(A)), and B has dimensions $(size(B)), but A and B must have the same size")) end - lda = max(1, stride(A, 2)) - ldb = max(1, stride(B, 2)) - alphar = similar(A, $elty, n) - alphai = similar(A, $elty, n) - beta = similar(A, $elty, n) ldvl = 0 if jobvl == 'V' ldvl = n @@ -2156,7 +2196,6 @@ for (geevx, ggev, ggev3, elty) in else throw(ArgumentError("jobvl must be 'V' or 'N', but $jobvl was passed")) end - vl = similar(A, $elty, ldvl, n) ldvr = 0 if jobvr == 'V' ldvr = n @@ -2165,6 +2204,12 @@ for (geevx, ggev, ggev3, elty) in else throw(ArgumentError("jobvr must be 'V' or 'N', but $jobvr was passed")) end + lda = max(1, stride(A, 2)) + ldb = max(1, stride(B, 2)) + alphar = similar(A, $elty, n) + alphai = similar(A, $elty, n) + beta = similar(A, $elty, n) + vl = similar(A, $elty, ldvl, n) vr = similar(A, $elty, ldvr, n) work = Vector{$elty}(undef, 1) lwork = BlasInt(-1) @@ -2207,11 +2252,6 @@ for (geevx, ggev, ggev3, elty) in if n != m throw(DimensionMismatch("A has dimensions $(size(A)), and B has dimensions $(size(B)), but A and B must have the same size")) end - lda = max(1, stride(A, 2)) - ldb = max(1, stride(B, 2)) - alphar = similar(A, $elty, n) - alphai = similar(A, $elty, n) - beta = similar(A, $elty, n) ldvl = 0 if jobvl == 'V' ldvl = n @@ -2220,7 +2260,6 @@ for (geevx, ggev, ggev3, elty) in else throw(ArgumentError("jobvl must be 'V' or 'N', but $jobvl was passed")) end - vl = similar(A, $elty, ldvl, n) ldvr = 0 if jobvr == 'V' ldvr = n @@ -2229,6 +2268,12 @@ for (geevx, ggev, ggev3, elty) in else throw(ArgumentError("jobvr must be 'V' or 'N', but $jobvr was passed")) end + lda = max(1, stride(A, 2)) + ldb = max(1, stride(B, 2)) + alphar = similar(A, $elty, n) + alphai = similar(A, $elty, n) + beta = similar(A, $elty, n) + vl = similar(A, $elty, ldvl, n) vr = similar(A, $elty, ldvr, n) work = Vector{$elty}(undef, 1) lwork = BlasInt(-1) @@ -2275,13 +2320,17 @@ for (geevx, ggev, ggev3, elty, relty) in # COMPLEX*16 A( LDA, * ), VL( LDVL, * ), VR( LDVR, * ), # $ W( * ), WORK( * ) function geevx!(balanc::AbstractChar, jobvl::AbstractChar, jobvr::AbstractChar, sense::AbstractChar, A::AbstractMatrix{$elty}) - n = checksquare(A) - chkfinite(A) # balancing routines don't support NaNs and Infs - lda = max(1,stride(A,2)) - w = similar(A, $elty, n) - if balanc ∉ ['N', 'P', 'S', 'B'] + require_one_based_indexing(A) + if balanc ∉ ('N', 'P', 'S', 'B') throw(ArgumentError("balanc must be 'N', 'P', 'S', or 'B', but $balanc was passed")) end + if sense ∉ ('N','E','V','B') + throw(ArgumentError("sense must be 'N', 'E', 'V' or 'B', but $sense was passed")) + end + if sense ∈ ('E', 'B') && !(jobvl == jobvr == 'V') + throw(ArgumentError("sense = '$sense' requires jobvl = 'V' and jobvr = 'V'")) + end + n = checksquare(A) ldvl = 0 if jobvl == 'V' ldvl = n @@ -2290,7 +2339,6 @@ for (geevx, ggev, ggev3, elty, relty) in else throw(ArgumentError("jobvl must be 'V' or 'N', but $jobvl was passed")) end - VL = similar(A, $elty, ldvl, n) ldvr = 0 if jobvr == 'V' ldvr = n @@ -2299,9 +2347,10 @@ for (geevx, ggev, ggev3, elty, relty) in else throw(ArgumentError("jobvr must be 'V' or 'N', but $jobvr was passed")) end - if sense ∉ ['N','E','V','B'] - throw(ArgumentError("sense must be 'N', 'E', 'V' or 'B', but $sense was passed")) - end + chkfinite(A) # balancing routines don't support NaNs and Infs + lda = max(1,stride(A,2)) + w = similar(A, $elty, n) + VL = similar(A, $elty, ldvl, n) VR = similar(A, $elty, ldvr, n) ilo = Ref{BlasInt}() ihi = Ref{BlasInt}() @@ -2354,10 +2403,6 @@ for (geevx, ggev, ggev3, elty, relty) in if n != m throw(DimensionMismatch("A has dimensions $(size(A)), and B has dimensions $(size(B)), but A and B must have the same size")) end - lda = max(1, stride(A, 2)) - ldb = max(1, stride(B, 2)) - alpha = similar(A, $elty, n) - beta = similar(A, $elty, n) ldvl = 0 if jobvl == 'V' ldvl = n @@ -2366,7 +2411,6 @@ for (geevx, ggev, ggev3, elty, relty) in else throw(ArgumentError("jobvl must be 'V' or 'N', but $jobvl was passed")) end - vl = similar(A, $elty, ldvl, n) ldvr = 0 if jobvr == 'V' ldvr = n @@ -2375,6 +2419,11 @@ for (geevx, ggev, ggev3, elty, relty) in else throw(ArgumentError("jobvr must be 'V' or 'N', but $jobvr was passed")) end + lda = max(1, stride(A, 2)) + ldb = max(1, stride(B, 2)) + alpha = similar(A, $elty, n) + beta = similar(A, $elty, n) + vl = similar(A, $elty, ldvl, n) vr = similar(A, $elty, ldvr, n) work = Vector{$elty}(undef, 1) lwork = BlasInt(-1) @@ -2419,10 +2468,6 @@ for (geevx, ggev, ggev3, elty, relty) in if n != m throw(DimensionMismatch("A has dimensions $(size(A)), and B has dimensions $(size(B)), but A and B must have the same size")) end - lda = max(1, stride(A, 2)) - ldb = max(1, stride(B, 2)) - alpha = similar(A, $elty, n) - beta = similar(A, $elty, n) ldvl = 0 if jobvl == 'V' ldvl = n @@ -2431,7 +2476,6 @@ for (geevx, ggev, ggev3, elty, relty) in else throw(ArgumentError("jobvl must be 'V' or 'N', but $jobvl was passed")) end - vl = similar(A, $elty, ldvl, n) ldvr = 0 if jobvr == 'V' ldvr = n @@ -2440,6 +2484,11 @@ for (geevx, ggev, ggev3, elty, relty) in else throw(ArgumentError("jobvr must be 'V' or 'N', but $jobvr was passed")) end + lda = max(1, stride(A, 2)) + ldb = max(1, stride(B, 2)) + alpha = similar(A, $elty, n) + beta = similar(A, $elty, n) + vl = similar(A, $elty, ldvl, n) vr = similar(A, $elty, ldvr, n) work = Vector{$elty}(undef, 1) lwork = BlasInt(-1) @@ -2524,6 +2573,7 @@ for (laic1, elty) in function laic1!(job::Integer, x::AbstractVector{$elty}, sest::$elty, w::AbstractVector{$elty}, gamma::$elty) require_one_based_indexing(x, w) + @chkvalidparam 1 job (1,2) j = length(x) if j != length(w) throw(DimensionMismatch("vectors must have same length, but length of x is $j and length of w is $(length(w))")) @@ -2558,6 +2608,7 @@ for (laic1, elty, relty) in function laic1!(job::Integer, x::AbstractVector{$elty}, sest::$relty, w::AbstractVector{$elty}, gamma::$elty) require_one_based_indexing(x, w) + @chkvalidparam 1 job (1,2) j = length(x) if j != length(w) throw(DimensionMismatch("vectors must have same length, but length of x is $j and length of w is $(length(w))")) @@ -3303,6 +3354,7 @@ for (posv, potrf, potri, potrs, pstrf, elty, rtyp) in # DOUBLE PRECISION A( LDA, * ), WORK( 2*N ) # INTEGER PIV( N ) function pstrf!(uplo::AbstractChar, A::AbstractMatrix{$elty}, tol::Real) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) chkuplo(uplo) @@ -3534,6 +3586,7 @@ for (trtri, trtrs, elty) in # .. Array Arguments .. # DOUBLE PRECISION A( LDA, * ) function trtri!(uplo::AbstractChar, diag::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) chkuplo(uplo) @@ -3616,10 +3669,12 @@ for (trcon, trevc, trrfs, elty) in # INTEGER IWORK( * ) # DOUBLE PRECISION A( LDA, * ), WORK( * ) function trcon!(norm::AbstractChar, uplo::AbstractChar, diag::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) chkstride1(A) chkdiag(diag) n = checksquare(A) chkuplo(uplo) + @chkvalidparam 1 norm ('O', '1', 'I') rcond = Ref{$elty}() work = Vector{$elty}(undef, 3n) iwork = Vector{BlasInt}(undef, n) @@ -3651,9 +3706,10 @@ for (trcon, trevc, trrfs, elty) in VR::AbstractMatrix{$elty} = similar(T)) require_one_based_indexing(select, T, VL, VR) # Extract - if side ∉ ['L','R','B'] + if side ∉ ('L','R','B') throw(ArgumentError("side argument must be 'L' (left eigenvectors), 'R' (right eigenvectors), or 'B' (both), got $side")) end + @chkvalidparam 2 howmny ('A', 'B', 'S') n, mm = checksquare(T), size(VL, 2) ldt, ldvl, ldvr = stride(T, 2), stride(VL, 2), stride(VR, 2) @@ -3749,8 +3805,10 @@ for (trcon, trevc, trrfs, elty, relty) in # DOUBLE PRECISION RWORK( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function trcon!(norm::AbstractChar, uplo::AbstractChar, diag::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) + @chkvalidparam 1 norm ('O', '1', 'I') chkuplo(uplo) chkdiag(diag) rcond = Ref{$relty}(1) @@ -3790,9 +3848,10 @@ for (trcon, trevc, trrfs, elty, relty) in # Check chkstride1(T, select, VL, VR) - if side ∉ ['L','R','B'] + if side ∉ ('L','R','B') throw(ArgumentError("side argument must be 'L' (left eigenvectors), 'R' (right eigenvectors), or 'B' (both), got $side")) end + @chkvalidparam 2 howmny ('A', 'B', 'S') # Allocate m = Ref{BlasInt}() @@ -3919,6 +3978,7 @@ for (stev, stebz, stegr, stein, elty) in @eval begin function stev!(job::AbstractChar, dv::AbstractVector{$elty}, ev::AbstractVector{$elty}) require_one_based_indexing(dv, ev) + @chkvalidparam 1 job ('N', 'V') chkstride1(dv, ev) n = length(dv) if length(ev) != n - 1 && length(ev) != n @@ -3941,6 +4001,8 @@ for (stev, stebz, stegr, stein, elty) in #* eigenvalues. function stebz!(range::AbstractChar, order::AbstractChar, vl::$elty, vu::$elty, il::Integer, iu::Integer, abstol::Real, dv::AbstractVector{$elty}, ev::AbstractVector{$elty}) require_one_based_indexing(dv, ev) + @chkvalidparam 1 range ('A', 'V', 'I') + @chkvalidparam 2 order ('B', 'E') chkstride1(dv, ev) n = length(dv) if length(ev) != n - 1 @@ -3972,6 +4034,8 @@ for (stev, stebz, stegr, stein, elty) in function stegr!(jobz::AbstractChar, range::AbstractChar, dv::AbstractVector{$elty}, ev::AbstractVector{$elty}, vl::Real, vu::Real, il::Integer, iu::Integer) require_one_based_indexing(dv, ev) + @chkvalidparam 1 jobz ('N', 'V') + @chkvalidparam 2 range ('A', 'V', 'I') chkstride1(dv, ev) n = length(dv) ne = length(ev) @@ -4144,6 +4208,7 @@ for (syconv, sysv, sytrf, sytri, sytrs, elty) in # INTEGER IPIV( * ) # DOUBLE PRECISION A( LDA, * ), WORK( * ) function syconv!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A, ipiv) chkstride1(A, ipiv) n = checksquare(A) chkuplo(uplo) @@ -4201,6 +4266,7 @@ for (syconv, sysv, sytrf, sytri, sytrs, elty) in # INTEGER IPIV( * ) # DOUBLE PRECISION A( LDA, * ), WORK( * ) function sytrf!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) chkuplo(uplo) @@ -4225,6 +4291,8 @@ for (syconv, sysv, sytrf, sytri, sytrs, elty) in end function sytrf!(uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) + chkuplo(uplo) n = checksquare(A) ipiv = similar(A, BlasInt, n) sytrf!(uplo, A, ipiv) @@ -4267,6 +4335,7 @@ for (syconv, sysv, sytrf, sytri, sytrs, elty) in # INTEGER IPIV( * ) # DOUBLE PRECISION A( LDA, * ), WORK( * ) function sytri!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A, ipiv) chkstride1(A, ipiv) n = checksquare(A) chkuplo(uplo) @@ -4358,6 +4427,7 @@ for (sysv, sytrf, sytri, sytrs, syconvf, elty) in # INTEGER IPIV( * ) # DOUBLE PRECISION A( LDA, * ), WORK( * ) function sytrf_rook!(uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) chkuplo(uplo) @@ -4390,6 +4460,7 @@ for (sysv, sytrf, sytri, sytrs, syconvf, elty) in # INTEGER IPIV( * ) # DOUBLE PRECISION A( LDA, * ), WORK( * ) function sytri_rook!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A, ipiv) chkstride1(A, ipiv) n = checksquare(A) chkuplo(uplo) @@ -4492,6 +4563,7 @@ for (syconv, hesv, hetrf, hetri, hetrs, elty, relty) in # INTEGER IPIV( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function syconv!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A,ipiv) chkstride1(A,ipiv) n = checksquare(A) chkuplo(uplo) @@ -4549,6 +4621,7 @@ for (syconv, hesv, hetrf, hetri, hetrs, elty, relty) in # INTEGER IPIV( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function hetrf!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) chkuplo(uplo) @@ -4570,6 +4643,8 @@ for (syconv, hesv, hetrf, hetri, hetrs, elty, relty) in end function hetrf!(uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) + chkuplo(uplo) n = checksquare(A) ipiv = similar(A, BlasInt, n) hetrf!(uplo, A, ipiv) @@ -4614,6 +4689,7 @@ for (syconv, hesv, hetrf, hetri, hetrs, elty, relty) in # INTEGER IPIV( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function hetri!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A, ipiv) chkstride1(A, ipiv) n = checksquare(A) chkuplo(uplo) @@ -4638,6 +4714,7 @@ for (syconv, hesv, hetrf, hetri, hetrs, elty, relty) in function hetrs!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}, B::AbstractVecOrMat{$elty}) require_one_based_indexing(A, ipiv, B) + chkuplo(uplo) chkstride1(A,B,ipiv) n = checksquare(A) if n != size(B,1) @@ -4702,6 +4779,7 @@ for (hesv, hetrf, hetri, hetrs, elty, relty) in # INTEGER IPIV( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function hetrf_rook!(uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) chkuplo(uplo) @@ -4732,6 +4810,7 @@ for (hesv, hetrf, hetri, hetrs, elty, relty) in # INTEGER IPIV( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function hetri_rook!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A,ipiv) chkstride1(A,ipiv) n = checksquare(A) chkuplo(uplo) @@ -4757,6 +4836,7 @@ for (hesv, hetrf, hetri, hetrs, elty, relty) in ipiv::AbstractVector{BlasInt}, B::AbstractVecOrMat{$elty}) require_one_based_indexing(A, ipiv, B) chkstride1(A,B,ipiv) + chkuplo(uplo) n = checksquare(A) if n != size(B,1) throw(DimensionMismatch("B has first dimension $(size(B,1)), but needs $n")) @@ -4822,6 +4902,7 @@ for (sysv, sytrf, sytri, sytrs, elty, relty) in # INTEGER IPIV( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function sytrf!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) chkuplo(uplo) @@ -4846,6 +4927,8 @@ for (sysv, sytrf, sytri, sytrs, elty, relty) in end function sytrf!(uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) + chkuplo(uplo) n = checksquare(A) ipiv = similar(A, BlasInt, n) sytrf!(uplo, A, ipiv) @@ -4889,6 +4972,7 @@ for (sysv, sytrf, sytri, sytrs, elty, relty) in # INTEGER IPIV( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function sytri!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A, ipiv) chkstride1(A, ipiv) n = checksquare(A) chkuplo(uplo) @@ -4980,6 +5064,7 @@ for (sysv, sytrf, sytri, sytrs, syconvf, elty, relty) in # INTEGER IPIV( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function sytrf_rook!(uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) chkuplo(uplo) @@ -5013,6 +5098,7 @@ for (sysv, sytrf, sytri, sytrs, syconvf, elty, relty) in # INTEGER IPIV( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function sytri_rook!(uplo::AbstractChar, A::AbstractMatrix{$elty}, ipiv::AbstractVector{BlasInt}) + require_one_based_indexing(A, ipiv) chkstride1(A, ipiv) n = checksquare(A) chkuplo(uplo) @@ -5240,6 +5326,9 @@ for (syev, syevr, syevd, sygvd, elty) in # * .. Array Arguments .. # DOUBLE PRECISION A( LDA, * ), W( * ), WORK( * ) function syev!(jobz::AbstractChar, uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) + @chkvalidparam 1 jobz ('N', 'V') + chkuplo(uplo) chkstride1(A) n = checksquare(A) W = similar(A, $elty, n) @@ -5273,15 +5362,18 @@ for (syev, syevr, syevd, sygvd, elty) in # DOUBLE PRECISION A( LDA, * ), W( * ), WORK( * ), Z( LDZ, * ) function syevr!(jobz::AbstractChar, range::AbstractChar, uplo::AbstractChar, A::AbstractMatrix{$elty}, vl::AbstractFloat, vu::AbstractFloat, il::Integer, iu::Integer, abstol::AbstractFloat) + require_one_based_indexing(A) + @chkvalidparam 1 jobz ('N', 'V') + @chkvalidparam 2 range ('A', 'V', 'I') chkstride1(A) n = checksquare(A) - chkuplofinite(A, uplo) if range == 'I' && !(1 <= il <= iu <= n) throw(ArgumentError("illegal choice of eigenvalue indices (il = $il, iu = $iu), which must be between 1 and n = $n")) end if range == 'V' && vl >= vu throw(ArgumentError("lower boundary, $vl, must be less than upper boundary, $vu")) end + chkuplofinite(A, uplo) lda = stride(A,2) m = Ref{BlasInt}() W = similar(A, $elty, n) @@ -5334,6 +5426,8 @@ for (syev, syevr, syevd, sygvd, elty) in # INTEGER IWORK( * ) # DOUBLE PRECISION A( LDA, * ), W( * ), WORK( * ) function syevd!(jobz::AbstractChar, uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) + @chkvalidparam 1 jobz ('N', 'V') chkstride1(A) n = checksquare(A) chkuplofinite(A, uplo) @@ -5375,6 +5469,10 @@ for (syev, syevr, syevd, sygvd, elty) in # INTEGER IWORK( * ) # DOUBLE PRECISION A( LDA, * ), B( LDB, * ), W( * ), WORK( * ) function sygvd!(itype::Integer, jobz::AbstractChar, uplo::AbstractChar, A::AbstractMatrix{$elty}, B::AbstractMatrix{$elty}) + require_one_based_indexing(A, B) + @chkvalidparam 1 itype 1:3 + @chkvalidparam 2 jobz ('N', 'V') + chkuplo(uplo) chkstride1(A, B) n, m = checksquare(A, B) if n != m @@ -5425,6 +5523,8 @@ for (syev, syevr, syevd, sygvd, elty, relty) in # DOUBLE PRECISION RWORK( * ), W( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function syev!(jobz::AbstractChar, uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) + @chkvalidparam 1 jobz ('N', 'V') chkstride1(A) chkuplofinite(A, uplo) n = checksquare(A) @@ -5464,6 +5564,9 @@ for (syev, syevr, syevd, sygvd, elty, relty) in # COMPLEX*16 A( LDA, * ), WORK( * ), Z( LDZ, * ) function syevr!(jobz::AbstractChar, range::AbstractChar, uplo::AbstractChar, A::AbstractMatrix{$elty}, vl::AbstractFloat, vu::AbstractFloat, il::Integer, iu::Integer, abstol::AbstractFloat) + require_one_based_indexing(A) + @chkvalidparam 1 jobz ('N', 'V') + @chkvalidparam 2 range ('A', 'V', 'I') chkstride1(A) chkuplofinite(A, uplo) n = checksquare(A) @@ -5533,6 +5636,8 @@ for (syev, syevr, syevd, sygvd, elty, relty) in # DOUBLE PRECISION RWORK( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function syevd!(jobz::AbstractChar, uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) + @chkvalidparam 1 jobz ('N', 'V') chkstride1(A) chkuplofinite(A, uplo) n = checksquare(A) @@ -5578,6 +5683,9 @@ for (syev, syevr, syevd, sygvd, elty, relty) in # DOUBLE PRECISION RWORK( * ), W( * ) # COMPLEX*16 A( LDA, * ), B( LDB, * ), WORK( * ) function sygvd!(itype::Integer, jobz::AbstractChar, uplo::AbstractChar, A::AbstractMatrix{$elty}, B::AbstractMatrix{$elty}) + require_one_based_indexing(A, B) + @chkvalidparam 1 itype 1:3 + @chkvalidparam 2 jobz ('N', 'V') chkstride1(A, B) chkuplofinite(A, uplo) chkuplofinite(B, uplo) @@ -5827,6 +5935,8 @@ for (gecon, elty) in # INTEGER IWORK( * ) # DOUBLE PRECISION A( LDA, * ), WORK( * ) function gecon!(normtype::AbstractChar, A::AbstractMatrix{$elty}, anorm::$elty) + require_one_based_indexing(A) + @chkvalidparam 1 normtype ('0', '1', 'I') chkstride1(A) n = checksquare(A) lda = max(1, stride(A, 2)) @@ -5861,6 +5971,8 @@ for (gecon, elty, relty) in # DOUBLE PRECISION RWORK( * ) # COMPLEX*16 A( LDA, * ), WORK( * ) function gecon!(normtype::AbstractChar, A::AbstractMatrix{$elty}, anorm::$relty) + require_one_based_indexing(A) + @chkvalidparam 1 normtype ('0', '1', 'I') chkstride1(A) n = checksquare(A) lda = max(1, stride(A, 2)) @@ -5903,6 +6015,7 @@ for (gehrd, elty) in # * .. Array Arguments .. # DOUBLE PRECISION A( LDA, * ), TAU( * ), WORK( * ) function gehrd!(ilo::Integer, ihi::Integer, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) chkfinite(A) # balancing routines don't support NaNs and Infs @@ -6005,6 +6118,8 @@ for (ormhr, elty) in require_one_based_indexing(A, tau, C) chkstride1(A, tau, C) + chkside(side) + chktrans(trans) n = checksquare(A) mC, nC = size(C, 1), size(C, 2) @@ -6052,6 +6167,8 @@ for (hseqr, elty) in function hseqr!(job::AbstractChar, compz::AbstractChar, ilo::Integer, ihi::Integer, H::AbstractMatrix{$elty}, Z::AbstractMatrix{$elty}) require_one_based_indexing(H, Z) + @chkvalidparam 1 job ('E', 'S') + @chkvalidparam 2 compz ('N', 'I', 'V') chkstride1(H) n = checksquare(H) checksquare(Z) == n || throw(DimensionMismatch()) @@ -6094,6 +6211,8 @@ for (hseqr, elty) in function hseqr!(job::AbstractChar, compz::AbstractChar, ilo::Integer, ihi::Integer, H::AbstractMatrix{$elty}, Z::AbstractMatrix{$elty}) require_one_based_indexing(H, Z) + @chkvalidparam 1 job ('E', 'S') + @chkvalidparam 2 compz ('N', 'I', 'V') chkstride1(H) n = checksquare(H) checksquare(Z) == n || throw(DimensionMismatch()) @@ -6152,6 +6271,7 @@ for (hetrd, elty) in # * .. Array Arguments .. # DOUBLE PRECISION A( LDA, * ), D( * ), E( * ), TAU( * ), WORK( * ) function hetrd!(uplo::AbstractChar, A::AbstractMatrix{$elty}) + require_one_based_indexing(A) chkstride1(A) n = checksquare(A) chkuplo(uplo) @@ -6257,7 +6377,9 @@ for (ormtr, elty) in require_one_based_indexing(A, tau, C) chkstride1(A, tau, C) n = checksquare(A) + chkside(side) chkuplo(uplo) + chktrans(trans) mC, nC = size(C, 1), size(C, 2) if n - length(tau) != 1 @@ -6305,6 +6427,7 @@ for (gees, gges, gges3, elty) in # $ WR( * ) function gees!(jobvs::AbstractChar, A::AbstractMatrix{$elty}) require_one_based_indexing(A) + @chkvalidparam 1 jobvs ('N', 'V') chkstride1(A) n = checksquare(A) sdim = Vector{BlasInt}(undef, 1) @@ -6344,6 +6467,9 @@ for (gees, gges, gges3, elty) in # $ B( LDB, * ), BETA( * ), VSL( LDVSL, * ), # $ VSR( LDVSR, * ), WORK( * ) function gges!(jobvsl::AbstractChar, jobvsr::AbstractChar, A::AbstractMatrix{$elty}, B::AbstractMatrix{$elty}) + require_one_based_indexing(A, B) + @chkvalidparam 1 jobvsl ('N', 'V') + @chkvalidparam 2 jobvsr ('N', 'V') chkstride1(A, B) n, m = checksquare(A, B) if n != m @@ -6393,6 +6519,9 @@ for (gees, gges, gges3, elty) in # $ B( LDB, * ), BETA( * ), VSL( LDVSL, * ), # $ VSR( LDVSR, * ), WORK( * ) function gges3!(jobvsl::AbstractChar, jobvsr::AbstractChar, A::AbstractMatrix{$elty}, B::AbstractMatrix{$elty}) + require_one_based_indexing(A, B) + @chkvalidparam 1 jobvsl ('N', 'V') + @chkvalidparam 2 jobvsr ('N', 'V') chkstride1(A, B) n, m = checksquare(A, B) if n != m @@ -6448,6 +6577,7 @@ for (gees, gges, gges3, elty, relty) in # COMPLEX*16 A( LDA, * ), VS( LDVS, * ), W( * ), WORK( * ) function gees!(jobvs::AbstractChar, A::AbstractMatrix{$elty}) require_one_based_indexing(A) + @chkvalidparam 1 jobvs ('N', 'V') chkstride1(A) n = checksquare(A) sort = 'N' @@ -6489,6 +6619,9 @@ for (gees, gges, gges3, elty, relty) in # $ BETA( * ), VSL( LDVSL, * ), VSR( LDVSR, * ), # $ WORK( * ) function gges!(jobvsl::AbstractChar, jobvsr::AbstractChar, A::AbstractMatrix{$elty}, B::AbstractMatrix{$elty}) + require_one_based_indexing(A, B) + @chkvalidparam 1 jobvsl ('N', 'V') + @chkvalidparam 2 jobvsr ('N', 'V') chkstride1(A, B) n, m = checksquare(A, B) if n != m @@ -6539,6 +6672,9 @@ for (gees, gges, gges3, elty, relty) in # $ BETA( * ), VSL( LDVSL, * ), VSR( LDVSR, * ), # $ WORK( * ) function gges3!(jobvsl::AbstractChar, jobvsr::AbstractChar, A::AbstractMatrix{$elty}, B::AbstractMatrix{$elty}) + require_one_based_indexing(A, B) + @chkvalidparam 1 jobvsl ('N', 'V') + @chkvalidparam 2 jobvsr ('N', 'V') chkstride1(A, B) n, m = checksquare(A, B) if n != m @@ -6627,6 +6763,8 @@ for (trexc, trsen, tgsen, elty) in # * .. Array Arguments .. # DOUBLE PRECISION Q( LDQ, * ), T( LDT, * ), WORK( * ) function trexc!(compq::AbstractChar, ifst::BlasInt, ilst::BlasInt, T::AbstractMatrix{$elty}, Q::AbstractMatrix{$elty}) + require_one_based_indexing(T, Q) + @chkvalidparam 1 compq ('V', 'N') chkstride1(T, Q) n = checksquare(T) ldt = max(1, stride(T, 2)) @@ -6659,6 +6797,9 @@ for (trexc, trsen, tgsen, elty) in # DOUBLE PRECISION Q( LDQ, * ), T( LDT, * ), WI( * ), WORK( * ), WR( * ) function trsen!(job::AbstractChar, compq::AbstractChar, select::AbstractVector{BlasInt}, T::AbstractMatrix{$elty}, Q::AbstractMatrix{$elty}) + require_one_based_indexing(T, Q, select) + @chkvalidparam 1 job ('N', 'E', 'V', 'B') + @chkvalidparam 2 compq ('V', 'N') chkstride1(T, Q, select) n = checksquare(T) ldt = max(1, stride(T, 2)) @@ -6714,6 +6855,7 @@ for (trexc, trsen, tgsen, elty) in # .. function tgsen!(select::AbstractVector{BlasInt}, S::AbstractMatrix{$elty}, T::AbstractMatrix{$elty}, Q::AbstractMatrix{$elty}, Z::AbstractMatrix{$elty}) + require_one_based_indexing(select, S, T, Q, Z) chkstride1(select, S, T, Q, Z) n, nt, nq, nz = checksquare(S, T, Q, Z) if n != nt @@ -6779,6 +6921,8 @@ for (trexc, trsen, tgsen, elty, relty) in # .. Array Arguments .. # DOUBLE PRECISION Q( LDQ, * ), T( LDT, * ), WORK( * ) function trexc!(compq::AbstractChar, ifst::BlasInt, ilst::BlasInt, T::AbstractMatrix{$elty}, Q::AbstractMatrix{$elty}) + require_one_based_indexing(T, Q) + @chkvalidparam 1 compq ('V', 'N') chkstride1(T, Q) n = checksquare(T) ldt = max(1, stride(T, 2)) @@ -6809,6 +6953,9 @@ for (trexc, trsen, tgsen, elty, relty) in # COMPLEX Q( LDQ, * ), T( LDT, * ), W( * ), WORK( * ) function trsen!(job::AbstractChar, compq::AbstractChar, select::AbstractVector{BlasInt}, T::AbstractMatrix{$elty}, Q::AbstractMatrix{$elty}) + require_one_based_indexing(select, T, Q) + @chkvalidparam 1 job ('N', 'E', 'V', 'B') + @chkvalidparam 2 compq ('N', 'V') chkstride1(select, T, Q) n = checksquare(T) ldt = max(1, stride(T, 2)) @@ -6859,6 +7006,7 @@ for (trexc, trsen, tgsen, elty, relty) in # .. function tgsen!(select::AbstractVector{BlasInt}, S::AbstractMatrix{$elty}, T::AbstractMatrix{$elty}, Q::AbstractMatrix{$elty}, Z::AbstractMatrix{$elty}) + require_one_based_indexing(select, S, T, Q, Z) chkstride1(select, S, T, Q, Z) n, nt, nq, nz = checksquare(S, T, Q, Z) if n != nt @@ -6959,6 +7107,8 @@ for (fn, elty, relty) in ((:dtrsyl_, :Float64, :Float64), function trsyl!(transa::AbstractChar, transb::AbstractChar, A::AbstractMatrix{$elty}, B::AbstractMatrix{$elty}, C::AbstractMatrix{$elty}, isgn::Int=1) require_one_based_indexing(A, B, C) + chktrans(transa) + chktrans(transb) chkstride1(A, B, C) m, n = checksquare(A), checksquare(B) lda = max(1, stride(A, 2)) diff --git a/stdlib/LinearAlgebra/test/lapack.jl b/stdlib/LinearAlgebra/test/lapack.jl index 6e12c85204a78..652c6c2e27e6c 100644 --- a/stdlib/LinearAlgebra/test/lapack.jl +++ b/stdlib/LinearAlgebra/test/lapack.jl @@ -9,6 +9,7 @@ using LinearAlgebra: BlasInt @test_throws ArgumentError LinearAlgebra.LAPACK.chkside('Z') @test_throws ArgumentError LinearAlgebra.LAPACK.chkdiag('Z') @test_throws ArgumentError LinearAlgebra.LAPACK.chktrans('Z') +@test_throws ArgumentError LinearAlgebra.LAPACK.chkvalidparam(1, "job", 2, (0,1)) @testset "syevr" begin Random.seed!(123) @@ -35,6 +36,12 @@ using LinearAlgebra: BlasInt @test vals_test ≈ vals @test Z_test*(Diagonal(vals)*Z_test') ≈ Asym @test_throws DimensionMismatch LAPACK.sygvd!(1, 'V', 'U', copy(Asym), zeros(elty, 6, 6)) + + @test_throws "jobz must be one of ('N', 'V'), but 'X' was passed" LAPACK.syevr!('X', Asym) + @test_throws "jobz must be one of ('N', 'V'), but 'X' was passed" LAPACK.syev!('X', 'U', Asym) + @test_throws "uplo argument must be 'U' (upper) or 'L' (lower), got 'M'" LAPACK.syev!('N', 'M', Asym) + @test_throws "jobz must be one of ('N', 'V'), but 'X' was passed" LAPACK.syevd!('X', 'U', Asym) + @test_throws "uplo argument must be 'U' (upper) or 'L' (lower), got 'M'" LAPACK.syevd!('N', 'M', Asym) end end @@ -112,7 +119,9 @@ end D = LAPACK.gbtrs!('N',2,1,6,AB,ipiv,D) A = diagm(-2 => dl2, -1 => dl, 0 => d, 1 => du) @test A\C ≈ D - @test_throws DimensionMismatch LAPACK.gbtrs!('N',2,1,6,AB,ipiv,Matrix{elty}(undef,7,6)) + M = Matrix{elty}(undef,7,6) + @test_throws DimensionMismatch LAPACK.gbtrs!('N',2,1,6,AB,ipiv,M) + @test_throws ArgumentError LAPACK.gbtrs!('M',2,1,6,AB,ipiv,M) @test_throws LinearAlgebra.LAPACKException LAPACK.gbtrf!(2,1,6,zeros(elty,6,6)) end end @@ -141,9 +150,11 @@ end x10, x11 = Vector{LinearAlgebra.BlasInt}.(undef, (10, 11)) @test_throws DimensionMismatch LAPACK.gels!('N',A10x10,B11x11) @test_throws DimensionMismatch LAPACK.gels!('T',A10x10,B11x11) + @test_throws ArgumentError LAPACK.gels!('X',A10x10,B11x11) @test_throws DimensionMismatch LAPACK.gesv!(A10x10,B11x11) @test_throws DimensionMismatch LAPACK.getrs!('N',A10x10,x10,B11x11) @test_throws DimensionMismatch LAPACK.getrs!('T',A10x10,x10,B11x11) + @test_throws ArgumentError LAPACK.getrs!('X',A10x10,x10,B11x11) @test_throws DimensionMismatch LAPACK.getri!(A10x10,x11) end end @@ -177,12 +188,20 @@ end @test U ≈ lU @test S ≈ lS @test V' ≈ lVt + @test_throws ArgumentError LAPACK.gesvd!('X','S',A) + @test_throws ArgumentError LAPACK.gesvd!('S','X',A) B = rand(elty,10,10) # xggsvd3 replaced xggsvd in LAPACK 3.6.0 if LAPACK.version() < v"3.6.0" - @test_throws DimensionMismatch LAPACK.ggsvd!('S','S','S',A,B) + @test_throws DimensionMismatch LAPACK.ggsvd!('N','N','N',A,B) + @test_throws ArgumentError LAPACK.ggsvd!('X','N','N',A,B) + @test_throws ArgumentError LAPACK.ggsvd!('N','X','N',A,B) + @test_throws ArgumentError LAPACK.ggsvd!('N','N','X',A,B) else - @test_throws DimensionMismatch LAPACK.ggsvd3!('S','S','S',A,B) + @test_throws DimensionMismatch LAPACK.ggsvd3!('N','N','N',A,B) + @test_throws ArgumentError LAPACK.ggsvd3!('X','N','N',A,B) + @test_throws ArgumentError LAPACK.ggsvd3!('N','X','N',A,B) + @test_throws ArgumentError LAPACK.ggsvd3!('N','N','X',A,B) end end end @@ -224,6 +243,7 @@ end X = rand(elty,10) B,Y,z = LAPACK.gels!('N',copy(A),copy(X)) @test A\X ≈ Y + @test_throws ArgumentError LAPACK.gels!('X',A,X) end end @@ -252,6 +272,9 @@ end fA = eigen(A, sortby=nothing) @test fA.values ≈ Aw @test fA.vectors ≈ Avr + + @test_throws ArgumentError LAPACK.geev!('X','V',A) + @test_throws ArgumentError LAPACK.geev!('N','X',A) end end @@ -284,6 +307,7 @@ end @test_throws DimensionMismatch LAPACK.gttrs!('N', x11, d, du, x9, y10, b) @test_throws DimensionMismatch LAPACK.gttrs!('N', dl, d, x11, x9, y10, b) @test_throws DimensionMismatch LAPACK.gttrs!('N', dl, d, du, x9, y10, x11) + @test_throws ArgumentError LAPACK.gttrs!('X', dl, d, du, x9, y10, x11) A = lu(Tridiagonal(dl,d,du)) b = rand(elty,10,5) c = copy(b) @@ -298,10 +322,17 @@ end A = rand(elty,10,10) A,tau = LAPACK.gelqf!(A) @test_throws DimensionMismatch LAPACK.orglq!(A,tau,11) - @test_throws DimensionMismatch LAPACK.ormlq!('R','N',A,tau,rand(elty,11,11)) - @test_throws DimensionMismatch LAPACK.ormlq!('L','N',A,tau,rand(elty,11,11)) - @test_throws DimensionMismatch LAPACK.ormlq!('R','N',A,zeros(elty,11),rand(elty,10,10)) - @test_throws DimensionMismatch LAPACK.ormlq!('L','N',A,zeros(elty,11),rand(elty,10,10)) + temp = rand(elty,11,11) + @test_throws DimensionMismatch LAPACK.ormlq!('R','N',A,tau,temp) + @test_throws DimensionMismatch LAPACK.ormlq!('L','N',A,tau,temp) + @test_throws ArgumentError LAPACK.ormlq!('X','N',A,tau,temp) + @test_throws ArgumentError LAPACK.ormlq!('R','X',A,tau,temp) + temp = zeros(elty,11) + B = copy(A) + @test_throws DimensionMismatch LAPACK.ormlq!('R','N',A,temp,B) + @test_throws DimensionMismatch LAPACK.ormlq!('L','N',A,temp,B) + @test_throws ArgumentError LAPACK.ormlq!('X','N',A,temp,B) + @test_throws ArgumentError LAPACK.ormlq!('L','X',A,temp,B) B = copy(A) C = LAPACK.orglq!(B,tau) @@ -312,30 +343,51 @@ end @test_throws DimensionMismatch LAPACK.orgqr!(A,tau,11) B = copy(A) @test LAPACK.orgqr!(B,tau) ≈ LAPACK.ormqr!('R','N',A,tau,Matrix{elty}(I, 10, 10)) - @test_throws DimensionMismatch LAPACK.ormqr!('R','N',A,tau,rand(elty,11,11)) - @test_throws DimensionMismatch LAPACK.ormqr!('L','N',A,tau,rand(elty,11,11)) - @test_throws DimensionMismatch LAPACK.ormqr!('R','N',A,zeros(elty,11),rand(elty,10,10)) - @test_throws DimensionMismatch LAPACK.ormqr!('L','N',A,zeros(elty,11),rand(elty,10,10)) + temp = rand(elty,11,11) + @test_throws DimensionMismatch LAPACK.ormqr!('R','N',A,tau,temp) + @test_throws DimensionMismatch LAPACK.ormqr!('L','N',A,tau,temp) + @test_throws ArgumentError LAPACK.ormqr!('X','N',A,tau,temp) + @test_throws ArgumentError LAPACK.ormqr!('L','X',A,tau,temp) + B = copy(A) + temp = zeros(elty,11) + @test_throws DimensionMismatch LAPACK.ormqr!('R','N',A,temp,B) + @test_throws DimensionMismatch LAPACK.ormqr!('L','N',A,temp,B) + @test_throws ArgumentError LAPACK.ormqr!('X','N',A,temp,B) + @test_throws ArgumentError LAPACK.ormqr!('L','X',A,temp,B) A = rand(elty,10,10) A,tau = LAPACK.geqlf!(A) @test_throws DimensionMismatch LAPACK.orgql!(A,tau,11) B = copy(A) @test LAPACK.orgql!(B,tau) ≈ LAPACK.ormql!('R','N',A,tau,Matrix{elty}(I, 10, 10)) - @test_throws DimensionMismatch LAPACK.ormql!('R','N',A,tau,rand(elty,11,11)) - @test_throws DimensionMismatch LAPACK.ormql!('L','N',A,tau,rand(elty,11,11)) - @test_throws DimensionMismatch LAPACK.ormql!('R','N',A,zeros(elty,11),rand(elty,10,10)) - @test_throws DimensionMismatch LAPACK.ormql!('L','N',A,zeros(elty,11),rand(elty,10,10)) + temp = rand(elty,11,11) + @test_throws DimensionMismatch LAPACK.ormql!('R','N',A,tau,temp) + @test_throws DimensionMismatch LAPACK.ormql!('L','N',A,tau,temp) + @test_throws ArgumentError LAPACK.ormql!('X','N',A,tau,temp) + @test_throws ArgumentError LAPACK.ormql!('L','X',A,tau,temp) + temp = zeros(elty,11) + B = copy(A) + @test_throws DimensionMismatch LAPACK.ormql!('R','N',A,temp,B) + @test_throws DimensionMismatch LAPACK.ormql!('L','N',A,temp,B) + @test_throws ArgumentError LAPACK.ormql!('X','N',A,temp,B) + @test_throws ArgumentError LAPACK.ormql!('L','X',A,temp,B) A = rand(elty,10,10) A,tau = LAPACK.gerqf!(A) @test_throws DimensionMismatch LAPACK.orgrq!(A,tau,11) B = copy(A) @test LAPACK.orgrq!(B,tau) ≈ LAPACK.ormrq!('R','N',A,tau,Matrix{elty}(I, 10, 10)) - @test_throws DimensionMismatch LAPACK.ormrq!('R','N',A,tau,rand(elty,11,11)) - @test_throws DimensionMismatch LAPACK.ormrq!('L','N',A,tau,rand(elty,11,11)) - @test_throws DimensionMismatch LAPACK.ormrq!('R','N',A,zeros(elty,11),rand(elty,10,10)) - @test_throws DimensionMismatch LAPACK.ormrq!('L','N',A,zeros(elty,11),rand(elty,10,10)) + temp = rand(elty,11,11) + @test_throws DimensionMismatch LAPACK.ormrq!('R','N',A,tau,temp) + @test_throws DimensionMismatch LAPACK.ormrq!('L','N',A,tau,temp) + @test_throws ArgumentError LAPACK.ormrq!('X','N',A,tau,temp) + @test_throws ArgumentError LAPACK.ormrq!('L','X',A,tau,temp) + B = copy(A) + temp = zeros(elty,11) + @test_throws DimensionMismatch LAPACK.ormrq!('R','N',A,temp,B) + @test_throws DimensionMismatch LAPACK.ormrq!('L','N',A,temp,B) + @test_throws ArgumentError LAPACK.ormrq!('X','N',A,temp,B) + @test_throws ArgumentError LAPACK.ormrq!('L','X',A,temp,B) A = rand(elty,10,11) Q = copy(A) @@ -351,21 +403,29 @@ end T = zeros(elty,10,11) @test_throws DimensionMismatch LAPACK.gemqrt!('L','N',V,T,C) @test_throws DimensionMismatch LAPACK.gemqrt!('R','N',V,T,C) + @test_throws ArgumentError LAPACK.gemqrt!('X','N',V,T,C) + @test_throws ArgumentError LAPACK.gemqrt!('R','X',V,T,C) C = rand(elty,10,10) V = rand(elty,11,10) T = zeros(elty,10,10) @test_throws DimensionMismatch LAPACK.gemqrt!('R','N',V,T,C) @test_throws DimensionMismatch LAPACK.gemqrt!('L','N',V,T,C) + @test_throws ArgumentError LAPACK.gemqrt!('X','N',V,T,C) + @test_throws ArgumentError LAPACK.gemqrt!('L','X',V,T,C) # test size(T) = (nb,k) ensures 1 <= nb <= k T = zeros(elty,10,10) V = rand(elty,5,10) @test_throws DimensionMismatch LAPACK.gemqrt!('L','N',V,T,C) + @test_throws ArgumentError LAPACK.gemqrt!('X','N',V,T,C) + @test_throws ArgumentError LAPACK.gemqrt!('L','X',V,T,C) C = rand(elty,10,10) V = rand(elty,10,10) T = zeros(elty,11,10) @test_throws DimensionMismatch LAPACK.gemqrt!('R','N',V,T,C) + @test_throws ArgumentError LAPACK.gemqrt!('X','N',V,T,C) + @test_throws ArgumentError LAPACK.gemqrt!('R','X',V,T,C) @test_throws DimensionMismatch LAPACK.orghr!(1, 10, C, zeros(elty,11)) end @@ -377,8 +437,12 @@ end A = A + transpose(A) #symmetric! B = copy(A) B,ipiv = LAPACK.sytrf!('U',B) + @test_throws ArgumentError LAPACK.sytrf!('X',B) @test triu(inv(A)) ≈ triu(LAPACK.sytri!('U',B,ipiv)) rtol=eps(cond(A)) - @test_throws DimensionMismatch LAPACK.sytrs!('U',B,ipiv,rand(elty,11,5)) + @test_throws ArgumentError LAPACK.sytri!('X',B,ipiv) + temp = rand(elty,11,5) + @test_throws DimensionMismatch LAPACK.sytrs!('U',B,ipiv,temp) + @test_throws ArgumentError LAPACK.sytrs!('X',B,ipiv,temp) @test LAPACK.sytrf!('U',zeros(elty,0,0)) == (zeros(elty,0,0),zeros(BlasInt,0),zero(BlasInt)) end @@ -389,7 +453,10 @@ end B = copy(A) B,ipiv = LAPACK.sytrf_rook!('U', B) @test triu(inv(A)) ≈ triu(LAPACK.sytri_rook!('U', B, ipiv)) rtol=eps(cond(A)) - @test_throws DimensionMismatch LAPACK.sytrs_rook!('U', B, ipiv, rand(elty, 11, 5)) + @test_throws ArgumentError LAPACK.sytri_rook!('X', B, ipiv) + temp = rand(elty, 11, 5) + @test_throws DimensionMismatch LAPACK.sytrs_rook!('U', B, ipiv, temp) + @test_throws ArgumentError LAPACK.sytrs_rook!('X', B, ipiv, temp) @test LAPACK.sytrf_rook!('U',zeros(elty, 0, 0)) == (zeros(elty, 0, 0),zeros(BlasInt, 0),zero(BlasInt)) A = rand(elty, 10, 10) A = A + transpose(A) #symmetric! @@ -398,7 +465,9 @@ end cnd = cond(A) b,A = LAPACK.sysv_rook!('U', A, b) @test b ≈ c rtol=eps(cnd) - @test_throws DimensionMismatch LAPACK.sysv_rook!('U',A,rand(elty,11)) + temp = rand(elty,11) + @test_throws DimensionMismatch LAPACK.sysv_rook!('U',A,temp) + @test_throws ArgumentError LAPACK.sysv_rook!('X',A,temp) # syconvf_rook error handling # way argument is wrong @@ -416,8 +485,11 @@ end A = A + A' #hermitian! B = copy(A) B,ipiv = LAPACK.hetrf!('U',B) - @test_throws DimensionMismatch LAPACK.hetrs!('U',B,ipiv,rand(elty,11,5)) - @test_throws DimensionMismatch LAPACK.hetrs_rook!('U',B,ipiv,rand(elty,11,5)) + temp = rand(elty,11,5) + @test_throws DimensionMismatch LAPACK.hetrs!('U',B,ipiv,temp) + @test_throws ArgumentError LAPACK.hetrs!('X',B,ipiv,temp) + @test_throws DimensionMismatch LAPACK.hetrs_rook!('U',B,ipiv,temp) + @test_throws ArgumentError LAPACK.hetrs_rook!('X',B,ipiv,temp) end end @@ -425,11 +497,21 @@ end @testset for elty in (Float32, Float64) d = rand(elty,10) e = rand(elty,9) - @test_throws DimensionMismatch LAPACK.stev!('U',d,rand(elty,11)) - @test_throws DimensionMismatch LAPACK.stebz!('A','B',zero(elty),zero(elty),0,0,-1.,d,rand(elty,10)) - @test_throws DimensionMismatch LAPACK.stegr!('N','A',d,rand(elty,11),zero(elty),zero(elty),0,0) - @test_throws DimensionMismatch LAPACK.stein!(d,zeros(elty,11),zeros(elty,10),zeros(BlasInt,10),zeros(BlasInt,10)) - @test_throws DimensionMismatch LAPACK.stein!(d,e,zeros(elty,11),zeros(BlasInt,10),zeros(BlasInt,10)) + temp = rand(elty,11) + @test_throws DimensionMismatch LAPACK.stev!('N',d,temp) + @test_throws ArgumentError LAPACK.stev!('X',d,temp) + temp = rand(elty,10) + @test_throws DimensionMismatch LAPACK.stebz!('A','B',zero(elty),zero(elty),0,0,-1.,d,temp) + @test_throws ArgumentError LAPACK.stebz!('X','B',zero(elty),zero(elty),0,0,-1.,d,temp) + @test_throws ArgumentError LAPACK.stebz!('A','X',zero(elty),zero(elty),0,0,-1.,d,temp) + temp11 = rand(elty,11) + @test_throws DimensionMismatch LAPACK.stegr!('N','A',d,temp11,zero(elty),zero(elty),0,0) + @test_throws ArgumentError LAPACK.stegr!('X','A',d,temp11,zero(elty),zero(elty),0,0) + @test_throws ArgumentError LAPACK.stegr!('N','X',d,temp11,zero(elty),zero(elty),0,0) + tempblasint10 = zeros(BlasInt,10) + tempblasint10_2 = zeros(BlasInt,10) + @test_throws DimensionMismatch LAPACK.stein!(d,temp11,temp,tempblasint10,tempblasint10_2) + @test_throws DimensionMismatch LAPACK.stein!(d,e,temp11,tempblasint10,tempblasint10_2) end end @@ -439,7 +521,13 @@ end A = triu(A) B = copy(A) @test inv(A) ≈ LAPACK.trtri!('U','N',B) - @test_throws DimensionMismatch LAPACK.trtrs!('U','N','N',B,zeros(elty,11,10)) + @test_throws ArgumentError LAPACK.trtri!('X','N',B) + @test_throws ArgumentError LAPACK.trtri!('U','X',B) + temp = zeros(elty,11,10) + @test_throws DimensionMismatch LAPACK.trtrs!('U','N','N',B,temp) + @test_throws ArgumentError LAPACK.trtrs!('X','N','N',B,temp) + @test_throws ArgumentError LAPACK.trtrs!('U','X','N',B,temp) + @test_throws ArgumentError LAPACK.trtrs!('U','N','X',B,temp) end end @@ -479,6 +567,8 @@ end LinearAlgebra.LAPACK.larf!('L', v, τ, C1) LinearAlgebra.LAPACK.larf!('R', v, conj(τ), C2) @test C ≈ C2*C1 + + @test_throws ArgumentError LAPACK.larf!('X', v, τ, C1) end end @@ -489,6 +579,8 @@ end @test_throws DimensionMismatch LAPACK.tgsen!(zeros(BlasInt,10),Z,Z,zeros(elty,11,11),Z) @test_throws DimensionMismatch LAPACK.tgsen!(zeros(BlasInt,10),Z,Z,Z,zeros(elty,11,11)) @test_throws DimensionMismatch LAPACK.trsyl!('N','N',Z,Z,zeros(elty,11,11)) + @test_throws ArgumentError LAPACK.trsyl!('X','N',Z,Z,zeros(elty,11,11)) + @test_throws ArgumentError LAPACK.trsyl!('N','X',Z,Z,zeros(elty,11,11)) @test_throws DimensionMismatch LAPACK.tzrzf!(zeros(elty,10,5)) A = triu(rand(elty,4,4)) @@ -508,6 +600,7 @@ end b,A = LAPACK.sysv!('U',A,b) @test b ≈ c @test_throws DimensionMismatch LAPACK.sysv!('U',A,rand(elty,11)) + @test_throws ArgumentError LAPACK.sysv!('X',A,rand(elty,11)) end end @@ -520,14 +613,17 @@ end c = A \ b b,A = LAPACK.hesv!('U',A,b) @test b ≈ c - @test_throws DimensionMismatch LAPACK.hesv!('U',A,rand(elty,11)) + temp = rand(elty,11) + @test_throws DimensionMismatch LAPACK.hesv!('U',A,temp) + @test_throws ArgumentError LAPACK.hesv!('X',A,temp) A = rand(elty,10,10) A = A + A' #hermitian! b = rand(elty,10) c = A \ b b,A = LAPACK.hesv_rook!('U',A,b) @test b ≈ c - @test_throws DimensionMismatch LAPACK.hesv_rook!('U',A,rand(elty,11)) + @test_throws DimensionMismatch LAPACK.hesv_rook!('U',A,temp) + @test_throws ArgumentError LAPACK.hesv_rook!('X',A,temp) end end @@ -563,8 +659,12 @@ end C = copy(B) if elty <: Complex @test A\B ≈ LAPACK.pttrs!('U',rdv,ev,C) - @test_throws DimensionMismatch LAPACK.pttrs!('U',rdv,Vector{elty}(undef,10),C) - @test_throws DimensionMismatch LAPACK.pttrs!('U',rdv,ev,Matrix{elty}(undef,11,11)) + tempvec = Vector{elty}(undef,10) + tempmat = Matrix{elty}(undef,11,11) + @test_throws DimensionMismatch LAPACK.pttrs!('U',rdv,tempvec,C) + @test_throws DimensionMismatch LAPACK.pttrs!('U',rdv,ev,tempmat) + @test_throws ArgumentError LAPACK.pttrs!('X',rdv,tempvec,C) + @test_throws ArgumentError LAPACK.pttrs!('X',rdv,ev,tempmat) else @test A\B ≈ LAPACK.pttrs!(rdv,ev,C) @test_throws DimensionMismatch LAPACK.pttrs!(rdv,Vector{elty}(undef,10),C) @@ -591,6 +691,8 @@ end offsizemat = Matrix{elty}(undef, n+1, n+1) @test_throws DimensionMismatch LAPACK.posv!('U', D, offsizemat) @test_throws DimensionMismatch LAPACK.potrs!('U', D, offsizemat) + @test_throws ArgumentError LAPACK.posv!('X', D, offsizemat) + @test_throws ArgumentError LAPACK.potrs!('X', D, offsizemat) @test LAPACK.potrs!('U',Matrix{elty}(undef,0,0),elty[]) == elty[] end @@ -613,6 +715,10 @@ end B = rand(elty,11,11) @test_throws DimensionMismatch LAPACK.gges!('V','V',A,B) @test_throws DimensionMismatch LAPACK.gges3!('V','V',A,B) + @test_throws ArgumentError LAPACK.gges!('X','V',A,B) + @test_throws ArgumentError LAPACK.gges3!('X','V',A,B) + @test_throws ArgumentError LAPACK.gges!('V','X',A,B) + @test_throws ArgumentError LAPACK.gges3!('V','X',A,B) end end @@ -632,8 +738,14 @@ end select,Vln,Vrn = LAPACK.trevc!('B','S',select,copy(T)) @test Vrn ≈ v @test Vln ≈ Vl - @test_throws ArgumentError LAPACK.trevc!('V','S',select,copy(T)) - @test_throws DimensionMismatch LAPACK.trrfs!('U','N','N',T,rand(elty,10,10),rand(elty,10,11)) + @test_throws ArgumentError LAPACK.trevc!('V','S',select,T) + @test_throws ArgumentError LAPACK.trevc!('R','X',select,T) + temp1010 = rand(elty,10,10) + temp1011 = rand(elty,10,11) + @test_throws DimensionMismatch LAPACK.trrfs!('U','N','N',T,temp1010,temp1011) + @test_throws ArgumentError LAPACK.trrfs!('X','N','N',T,temp1010,temp1011) + @test_throws ArgumentError LAPACK.trrfs!('U','X','N',T,temp1010,temp1011) + @test_throws ArgumentError LAPACK.trrfs!('U','N','X',T,temp1010,temp1011) end end From 91da115ee35f555f0a61e4623379a1099c9e6abf Mon Sep 17 00:00:00 2001 From: Cameron Bieganek <8310743+CameronBieganek@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:24:51 -0500 Subject: [PATCH 46/55] Make some improvements to the Scoped Values documentation. (#53628) Fixes #53471. One thing to note, I changed the signature in the `with` docstring from this: ```julia with(f, (var::ScopedValue{T} => val::T)...) ``` to this: ```julia with(f, (var::ScopedValue{T} => val)...) ``` ...since the original signature in the docstring was too strict. I also added this sentence to the docstring: ```julia `val` will be converted to type `T`. ``` I added a couple tests that verify the conversion behavior. (cherry picked from commit 7613c6954fd2d9168f1c805d9212f961a4765a67) --- base/scopedvalues.jl | 101 ++++++++++++++++++++++++++++++++--- doc/src/base/scopedvalues.md | 93 +++++++++++++++++++------------- test/scopedvalues.jl | 10 ++++ 3 files changed, 159 insertions(+), 45 deletions(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index 339ee717d5fd6..6ccd4687c5c65 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -3,6 +3,7 @@ module ScopedValues export ScopedValue, with, @with +public get """ ScopedValue(x) @@ -54,7 +55,22 @@ Base.eltype(::ScopedValue{T}) where {T} = T """ isassigned(val::ScopedValue) -Test whether a ScopedValue has an assigned value. +Test whether a `ScopedValue` has an assigned value. + +See also: [`ScopedValues.with`](@ref), [`ScopedValues.@with`](@ref), [`ScopedValues.get`](@ref). + +# Examples +```jldoctest +julia> using Base.ScopedValues + +julia> a = ScopedValue(1); b = ScopedValue{Int}(); + +julia> isassigned(a) +true + +julia> isassigned(b) +false +``` """ function Base.isassigned(val::ScopedValue) val.has_default && return true @@ -114,6 +130,21 @@ const novalue = NoValue() If the scoped value isn't set and doesn't have a default value, return `nothing`. Otherwise returns `Some{T}` with the current value. + +See also: [`ScopedValues.with`](@ref), [`ScopedValues.@with`](@ref), [`ScopedValues.ScopedValue`](@ref). + +# Examples +```jldoctest +julia> using Base.ScopedValues + +julia> a = ScopedValue(42); b = ScopedValue{Int}(); + +julia> ScopedValues.get(a) +Some(42) + +julia> isnothing(ScopedValues.get(b)) +true +``` """ function get(val::ScopedValue{T}) where {T} scope = Core.current_scope()::Union{Scope, Nothing} @@ -151,11 +182,32 @@ function Base.show(io::IO, val::ScopedValue) end """ - @with vars... expr + @with (var::ScopedValue{T} => val)... expr + +Macro version of `with`. The expression `@with var=>val expr` evaluates `expr` in a +new dynamic scope with `var` set to `val`. `val` will be converted to type `T`. +`@with var=>val expr` is equivalent to `with(var=>val) do expr end`, but `@with` +avoids creating a closure. + +See also: [`ScopedValues.with`](@ref), [`ScopedValues.ScopedValue`](@ref), [`ScopedValues.get`](@ref). + +# Examples +```jldoctest +julia> using Base.ScopedValues -Macro version of `with(f, vars...)` but with `expr` instead of `f` function. -This is similar to using [`with`](@ref) with a `do` block, but avoids creating -a closure. +julia> const a = ScopedValue(1); + +julia> f(x) = a[] + x; + +julia> @with a=>2 f(10) +12 + +julia> @with a=>3 begin + x = 100 + f(x) + end +103 +``` """ macro with(exprs...) if length(exprs) > 1 @@ -172,9 +224,44 @@ macro with(exprs...) end """ - with(f, (var::ScopedValue{T} => val::T)...) + with(f, (var::ScopedValue{T} => val)...) + +Execute `f` in a new dynamic scope with `var` set to `val`. `val` will be converted +to type `T`. + +See also: [`ScopedValues.@with`](@ref), [`ScopedValues.ScopedValue`](@ref), [`ScopedValues.get`](@ref). + +# Examples +```jldoctest +julia> using Base.ScopedValues + +julia> a = ScopedValue(1); -Execute `f` in a new scope with `var` set to `val`. +julia> f(x) = a[] + x; + +julia> f(10) +11 + +julia> with(a=>2) do + f(10) + end +12 + +julia> f(10) +11 + +julia> b = ScopedValue(2); + +julia> g(x) = a[] + b[] + x; + +julia> with(a=>10, b=>20) do + g(30) + end +60 + +julia> with(() -> a[] * b[], a=>3, b=>4) +12 +``` """ function with(f, pair::Pair{<:ScopedValue}, rest::Pair{<:ScopedValue}...) @with(pair, rest..., f()) diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index 8c3340ab9cf11..6ad553429bb1f 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -17,18 +17,15 @@ concurrently. Scoped values were introduced in Julia 1.11. In Julia 1.8+ a compatible implementation is available from the package ScopedValues.jl. -In its simplest form you can create a [`Base.ScopedValue`](@ref) with a -default value and then use [`Base.with`](@ref with) or [`Base.@with`](@ref) to -enter a new dynamic scope. +In its simplest form you can create a [`ScopedValue`](@ref Base.ScopedValues.ScopedValue) +with a default value and then use [`with`](@ref Base.ScopedValues.with) or +[`@with`](@ref Base.ScopedValues.@with) to enter a new dynamic scope. The new scope will +inherit all values from the parent scope (and recursively from all outer scopes) with the +provided scoped value taking priority over previous definitions. -The new scope will inherit all values from the parent scope -(and recursively from all outer scopes) with the provided scoped -value taking priority over previous definitions. - -Let's first look at an example of **lexical** scope: - -A `let` statements begins a new lexical scope within which the outer definition -of `x` is shadowed by it's inner definition. +Let's first look at an example of **lexical** scope. A `let` statement begins +a new lexical scope within which the outer definition of `x` is shadowed by +it's inner definition. ```julia x = 1 @@ -38,9 +35,9 @@ end @show x # 1 ``` -Since Julia uses lexical scope the variable `x` is bound within the function `f` -to the global scope and entering a `let` scope does not change the value `f` -observes. +In the following example, since Julia uses lexical scope, the variable `x` in the body +of `f` refers to the `x` defined in the global scope, and entering a `let` scope does +not change the value `f` observes. ```julia x = 1 @@ -64,7 +61,7 @@ end f() # 1 ``` -Not that the observed value of the `ScopedValue` is dependent on the execution +Note that the observed value of the `ScopedValue` is dependent on the execution path of the program. It often makes sense to use a `const` variable to point to a scoped value, @@ -74,34 +71,54 @@ and you can set the value of multiple `ScopedValue`s with one call to `with`. ```julia using Base.ScopedValues -const scoped_val = ScopedValue(1) -const scoped_val2 = ScopedValue(0) - -# Enter a new dynamic scope and set value -@show scoped_val[] # 1 -@show scoped_val2[] # 0 -with(scoped_val => 2) do - @show scoped_val[] # 2 - @show scoped_val2[] # 0 - with(scoped_val => 3, scoped_val2 => 5) do - @show scoped_val[] # 3 - @show scoped_val2[] # 5 +f() = @show a[] +g() = @show b[] + +const a = ScopedValue(1) +const b = ScopedValue(2) + +f() # a[] = 1 +g() # b[] = 2 + +# Enter a new dynamic scope and set value. +with(a => 3) do + f() # a[] = 3 + g() # b[] = 2 + with(a => 4, b => 5) do + f() # a[] = 4 + g() # b[] = 5 end - @show scoped_val[] # 2 - @show scoped_val2[] # 0 + f() # a[] = 3 + g() # b[] = 2 end -@show scoped_val[] # 1 -@show scoped_val2[] # 0 + +f() # a[] = 1 +g() # b[] = 2 ``` -Since `with` requires a closure or a function and creates another call-frame, -it can sometimes be beneficial to use the macro form. +`ScopedValues` provides a macro version of `with`. The expression `@with var=>val expr` +evaluates `expr` in a new dynamic scope with `var` set to `val`. `@with var=>val expr` +is equivalent to `with(var=>val) do expr end`. However, `with` requires a zero-argument +closure or function, which results in an extra call-frame. As an example, consider the +following function `f`: ```julia using Base.ScopedValues +const a = ScopedValue(1) +f(x) = a[] + x +``` + +If you wish to run `f` in a dynamic scope with `a` set to `2`, then you can use `with`: -const STATE = ScopedValue{State}() -with_state(f, state::State) = @with(STATE => state, f()) +```julia +with(() -> f(10), a=>2) +``` + +However, this requires wrapping `f` in a zero-argument function. If you wish to avoid +the extra call-frame, then you can use the `@with` macro: + +```julia +@with a=>2 f(10) ``` !!! note @@ -265,11 +282,11 @@ Base.@kwdef struct Configuration verbose::Bool = false end -const CONFIG = ScopedValue(Configuration()) +const CONFIG = ScopedValue(Configuration(color=true)) -@with CONFIG => Configuration(CONFIG[], color=true) begin +@with CONFIG => Configuration(color=CONFIG[].color, verbose=true) begin @show CONFIG[].color # true - @show CONFIG[].verbose # false + @show CONFIG[].verbose # true end ``` diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index ca700521b50cd..3af6a3f065c8b 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -52,6 +52,16 @@ emptyf() = nothing @testset "conversion" begin with(emptyf, sval_float=>2) @test_throws MethodError with(emptyf, sval_float=>"hello") + a = ScopedValue(1) + with(a => 2.0) do + @test a[] == 2 + @test a[] isa Int + end + a = ScopedValue(1.0) + with(a => 2) do + @test a[] == 2.0 + @test a[] isa Float64 + end end import Base.Threads: @spawn From 02839bd09b9283821a316741fe485f8b124672bb Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 14 Mar 2024 10:51:10 -0400 Subject: [PATCH 47/55] codegen: change tbaa of ptr_phi to tbaa_value (#53655) Fixes https://github.com/JuliaLang/julia/issues/53612 (cherry picked from commit cb47b01d3eff8da054384bc0c293db8f9f8b2783) --- src/codegen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 6de2669ac0f7d..ad575d956e1e0 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5630,7 +5630,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) Value *ptr = ctx.builder.CreateSelect(isboxed, maybe_bitcast(ctx, decay_derived(ctx, ptr_phi), getInt8PtrTy(ctx.builder.getContext())), maybe_bitcast(ctx, decay_derived(ctx, phi), getInt8PtrTy(ctx.builder.getContext()))); - jl_cgval_t val = mark_julia_slot(ptr, phiType, Tindex_phi, ctx.tbaa().tbaa_stack); // XXX: this TBAA is wrong for ptr_phi + jl_cgval_t val = mark_julia_slot(ptr, phiType, Tindex_phi, best_tbaa(ctx.tbaa(), phiType)); val.Vboxed = ptr_phi; ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, ptr_phi, r)); ctx.SAvalues[idx] = val; From 2e45f8f6bd6bec9b20b33b02f357a83829201eda Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 14 Mar 2024 15:30:40 -0400 Subject: [PATCH 48/55] Default to the medium code model in x86 linux (#53391) This shouldn't have any cost on smaller images because the only thing that gets put into ldata is the system image data, which is only reference via `dlsym`. This allows for images larger than 2gb (tested by putting a 2gb array in the base image) I did not test how this might be handled in other platforms (Windows doesn't support it). (cherry picked from commit 0f04b33e01d3139ea103d9153dc442418c206b8a) --- src/aotcompile.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 7b51579ce430f..c84ef4e756cb0 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -4,6 +4,7 @@ #include "platform.h" // target support +#include "llvm/Support/CodeGen.h" #include #include #include @@ -1615,10 +1616,11 @@ void jl_dump_native_impl(void *native_code, if (TheTriple.isOSLinux() || TheTriple.isOSFreeBSD()) { RelocModel = Reloc::PIC_; } + CodeModel::Model CMModel = CodeModel::Small; - if (TheTriple.isPPC()) { - // On PPC the small model is limited to 16bit offsets - CMModel = CodeModel::Medium; + if (TheTriple.isPPC() || (TheTriple.isX86() && TheTriple.isArch64Bit() && TheTriple.isOSLinux())) { + // On PPC the small model is limited to 16bit offsets. For very large images the small code model + CMModel = CodeModel::Medium; // isn't good enough on x86 so use Medium, it has no cost because only the image goes in .ldata } std::unique_ptr SourceTM( jl_ExecutionEngine->getTarget().createTargetMachine( @@ -1661,6 +1663,12 @@ void jl_dump_native_impl(void *native_code, GlobalVariable::ExternalLinkage, data, "jl_system_image_data"); sysdata->setAlignment(Align(64)); +#if JL_LLVM_VERSION >= 180000 + sysdata->setCodeModel(CodeModel::Large); +#else + if (TheTriple.isX86() && TheTriple.isArch64Bit() && TheTriple.isOSLinux()) + sysdata->setSection(".ldata"); +#endif addComdat(sysdata, TheTriple); Constant *len = ConstantInt::get(sysimgM.getDataLayout().getIntPtrType(Context), z->size); addComdat(new GlobalVariable(sysimgM, len->getType(), true, From 2623abb8cbfcfd697c49766616664f2e936ca261 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 14 Mar 2024 22:17:44 -0400 Subject: [PATCH 49/55] Enable analyzegc checks for try catch and fix found issues (#53527) This PR also makes a successful `JL_TRY` not do so much work + fixes clang not finding the sdk when running those tests in macos. Fixes https://github.com/JuliaLang/julia/issues/ Co-authored-by: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Co-authored-by: Jameson Nash (cherry picked from commit 67cdb9bf70a39e62992276cb3923e928c2e8b637) --- Make.inc | 2 +- base/stream.jl | 12 ++- doc/src/devdocs/ast.md | 2 +- src/ast.c | 4 +- src/ccall.cpp | 2 +- src/codegen.cpp | 94 ++++++++++++++------ src/gc.c | 2 +- src/gf.c | 4 +- src/init.c | 6 +- src/interpreter.c | 12 +-- src/jl_exported_funcs.inc | 2 + src/jlapi.c | 28 +++--- src/jloptions.c | 3 +- src/jltypes.c | 2 +- src/julia.h | 74 ++++++++------- src/julia_internal.h | 2 +- src/llvm-codegen-shared.h | 2 +- src/llvm-late-gc-lowering.cpp | 2 +- src/llvm-lower-handlers.cpp | 34 +++---- src/llvm-pass-helpers.cpp | 3 +- src/llvm-pass-helpers.h | 1 + src/method.c | 11 ++- src/rtutils.c | 42 +++++---- src/scheduler.c | 2 +- src/signals-mach.c | 6 +- src/task.c | 27 +++--- src/toplevel.c | 4 +- test/Makefile | 1 + test/llvmpasses/Makefile | 5 +- test/llvmpasses/lower-handlers-addrspaces.ll | 8 +- test/llvmpasses/lower-handlers.ll | 8 +- 31 files changed, 241 insertions(+), 166 deletions(-) diff --git a/Make.inc b/Make.inc index 5d58454c80333..2a23bd3d37b51 100644 --- a/Make.inc +++ b/Make.inc @@ -1487,7 +1487,7 @@ endif CLANGSA_FLAGS := CLANGSA_CXXFLAGS := ifeq ($(OS), Darwin) # on new XCode, the files are hidden -CLANGSA_FLAGS += -isysroot $(shell xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk + CLANGSA_FLAGS += -isysroot $(shell xcrun --show-sdk-path -sdk macosx) endif ifeq ($(USEGCC),1) # try to help clang find the c++ files for CC by guessing the value for --prefix diff --git a/base/stream.jl b/base/stream.jl index 3264b8f153677..d53f3a2957a36 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -453,9 +453,11 @@ function closewrite(s::LibuvStream) sigatomic_begin() uv_req_set_data(req, ct) iolock_end() - status = try + local status + try sigatomic_end() - wait()::Cint + status = wait()::Cint + sigatomic_begin() finally # try-finally unwinds the sigatomic level, so need to repeat sigatomic_end sigatomic_end() @@ -1061,12 +1063,14 @@ function uv_write(s::LibuvStream, p::Ptr{UInt8}, n::UInt) sigatomic_begin() uv_req_set_data(uvw, ct) iolock_end() - status = try + local status + try sigatomic_end() # wait for the last chunk to complete (or error) # assume that any errors would be sticky, # (so we don't need to monitor the error status of the intermediate writes) - wait()::Cint + status = wait()::Cint + sigatomic_begin() finally # try-finally unwinds the sigatomic level, so need to repeat sigatomic_end sigatomic_end() diff --git a/doc/src/devdocs/ast.md b/doc/src/devdocs/ast.md index c2cd0e92d58d2..e3d5e038ad41a 100644 --- a/doc/src/devdocs/ast.md +++ b/doc/src/devdocs/ast.md @@ -431,7 +431,7 @@ These symbols appear in the `head` field of [`Expr`](@ref)s in lowered form. * `the_exception` - Yields the caught exception inside a `catch` block, as returned by `jl_current_exception()`. + Yields the caught exception inside a `catch` block, as returned by `jl_current_exception(ct)`. * `enter` diff --git a/src/ast.c b/src/ast.c index 8816c905ee101..3137dc397a451 100644 --- a/src/ast.c +++ b/src/ast.c @@ -475,7 +475,7 @@ static jl_value_t *scm_to_julia(fl_context_t *fl_ctx, value_t e, jl_module_t *mo } JL_CATCH { // if expression cannot be converted, replace with error expr - //jl_(jl_current_exception()); + //jl_(jl_current_exception(ct)); //jlbacktrace(); jl_expr_t *ex = jl_exprn(jl_error_sym, 1); v = (jl_value_t*)ex; @@ -1151,7 +1151,7 @@ static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule margs[0] = jl_cstr_to_string(""); margs[1] = jl_fieldref(lno, 0); // extract and allocate line number jl_rethrow_other(jl_new_struct(jl_loaderror_type, margs[0], margs[1], - jl_current_exception())); + jl_current_exception(ct))); } } ct->world_age = last_age; diff --git a/src/ccall.cpp b/src/ccall.cpp index b71d86623065f..96f0526fc8cf2 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1613,7 +1613,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) assert(lrt == ctx.types().T_prjlvalue); assert(!isVa && !llvmcall && nccallargs == 0); JL_GC_POP(); - auto ct = track_pjlvalue(ctx, emit_bitcast(ctx, get_current_task(ctx), ctx.types().T_pjlvalue)); + auto ct = track_pjlvalue(ctx, get_current_task(ctx)); return mark_or_box_ccall_result(ctx, ct, retboxed, rt, unionall, static_rt); } else if (is_libjulia_func(jl_set_next_task)) { diff --git a/src/codegen.cpp b/src/codegen.cpp index ad575d956e1e0..4b8666c223c28 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1048,30 +1048,64 @@ static const auto jlunlockfield_func = new JuliaFunction<>{ }; static const auto jlenter_func = new JuliaFunction<>{ XSTR(jl_enter_handler), - [](LLVMContext &C) { return FunctionType::get(getVoidTy(C), - {getInt8PtrTy(C)}, false); }, + [](LLVMContext &C) { + auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); + return FunctionType::get(getVoidTy(C), + {T_pjlvalue, getInt8PtrTy(C)}, false); }, nullptr, }; static const auto jl_current_exception_func = new JuliaFunction<>{ XSTR(jl_current_exception), - [](LLVMContext &C) { return FunctionType::get(JuliaType::get_prjlvalue_ty(C), false); }, + [](LLVMContext &C) { return FunctionType::get(JuliaType::get_prjlvalue_ty(C), {JuliaType::get_pjlvalue_ty(C)}, false); }, nullptr, }; static const auto jlleave_func = new JuliaFunction<>{ XSTR(jl_pop_handler), - [](LLVMContext &C) { return FunctionType::get(getVoidTy(C), - {getInt32Ty(C)}, false); }, - nullptr, + [](LLVMContext &C) { + auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); + return FunctionType::get(getVoidTy(C), + {T_pjlvalue, getInt32Ty(C)}, false); }, + [](LLVMContext &C) { + auto FnAttrs = AttrBuilder(C); + FnAttrs.addAttribute(Attribute::WillReturn); + FnAttrs.addAttribute(Attribute::NoUnwind); + auto RetAttrs = AttrBuilder(C); + return AttributeList::get(C, + AttributeSet::get(C, FnAttrs), + AttributeSet(), + None); + }, +}; +static const auto jlleave_noexcept_func = new JuliaFunction<>{ + XSTR(jl_pop_handler_noexcept), + [](LLVMContext &C) { + auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); + return FunctionType::get(getVoidTy(C), + {T_pjlvalue, getInt32Ty(C)}, false); }, + [](LLVMContext &C) { + auto FnAttrs = AttrBuilder(C); + FnAttrs.addAttribute(Attribute::WillReturn); + FnAttrs.addAttribute(Attribute::NoUnwind); + auto RetAttrs = AttrBuilder(C); + return AttributeList::get(C, + AttributeSet::get(C, FnAttrs), + AttributeSet(), + None); + }, }; static const auto jl_restore_excstack_func = new JuliaFunction{ XSTR(jl_restore_excstack), - [](LLVMContext &C, Type *T_size) { return FunctionType::get(getVoidTy(C), - {T_size}, false); }, + [](LLVMContext &C, Type *T_size) { + auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); + return FunctionType::get(getVoidTy(C), + {T_pjlvalue, T_size}, false); }, nullptr, }; static const auto jl_excstack_state_func = new JuliaFunction{ XSTR(jl_excstack_state), - [](LLVMContext &C, Type *T_size) { return FunctionType::get(T_size, false); }, + [](LLVMContext &C, Type *T_size) { + auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); + return FunctionType::get(T_size, {T_pjlvalue}, false); }, nullptr, }; static const auto jlegalx_func = new JuliaFunction{ @@ -1098,9 +1132,9 @@ static const auto jl_alloc_obj_func = new JuliaFunction{ [](LLVMContext &C, Type *T_size) { auto T_jlvalue = JuliaType::get_jlvalue_ty(C); auto T_prjlvalue = PointerType::get(T_jlvalue, AddressSpace::Tracked); - auto T_ppjlvalue = PointerType::get(PointerType::get(T_jlvalue, 0), 0); + auto T_pjlvalue = PointerType::get(T_jlvalue, 0); return FunctionType::get(T_prjlvalue, - {T_ppjlvalue, T_size, T_prjlvalue}, false); + {T_pjlvalue, T_size, T_prjlvalue}, false); }, [](LLVMContext &C) { auto FnAttrs = AttrBuilder(C); @@ -1442,7 +1476,9 @@ static const auto gc_preserve_end_func = new JuliaFunction<> { }; static const auto except_enter_func = new JuliaFunction<>{ "julia.except_enter", - [](LLVMContext &C) { return FunctionType::get(getInt32Ty(C), false); }, + [](LLVMContext &C) { + auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); + return FunctionType::get(getInt32Ty(C), {T_pjlvalue}, false); }, [](LLVMContext &C) { return AttributeList::get(C, Attributes(C, {Attribute::ReturnsTwice}), AttributeSet(), @@ -5957,8 +5993,7 @@ static void emit_stmtpos(jl_codectx_t &ctx, jl_value_t *expr, int ssaval_result) hand_n_leave += 1; } } - ctx.builder.CreateCall(prepare_call(jlleave_func), - ConstantInt::get(getInt32Ty(ctx.builder.getContext()), hand_n_leave)); + ctx.builder.CreateCall(prepare_call(jlleave_noexcept_func), {get_current_task(ctx), ConstantInt::get(getInt32Ty(ctx.builder.getContext()), hand_n_leave)}); if (scope_to_restore) { jl_aliasinfo_t scope_ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); scope_ai.decorateInst( @@ -5968,7 +6003,7 @@ static void emit_stmtpos(jl_codectx_t &ctx, jl_value_t *expr, int ssaval_result) else if (head == jl_pop_exception_sym) { jl_cgval_t excstack_state = emit_expr(ctx, jl_exprarg(expr, 0)); assert(excstack_state.V && excstack_state.V->getType() == ctx.types().T_size); - ctx.builder.CreateCall(prepare_call(jl_restore_excstack_func), excstack_state.V); + ctx.builder.CreateCall(prepare_call(jl_restore_excstack_func), {get_current_task(ctx), excstack_state.V}); return; } else { @@ -6199,7 +6234,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ bnd = jl_get_binding_for_method_def(mod, (jl_sym_t*)mn); } JL_CATCH { - jl_value_t *e = jl_current_exception(); + jl_value_t *e = jl_current_exception(jl_current_task); // errors. boo. :( JL_GC_PUSH1(&e); e = jl_as_global_root(e, 1); @@ -6375,7 +6410,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ else if (head == jl_exc_sym) { assert(nargs == 0); return mark_julia_type(ctx, - ctx.builder.CreateCall(prepare_call(jl_current_exception_func)), + ctx.builder.CreateCall(prepare_call(jl_current_exception_func), {get_current_task(ctx)}), true, jl_any_type); } else if (head == jl_copyast_sym) { @@ -6479,9 +6514,14 @@ static void allocate_gc_frame(jl_codectx_t &ctx, BasicBlock *b0, bool or_new=fal ctx.pgcstack->setName("pgcstack"); } +static Value *get_current_task(jl_codectx_t &ctx, Type *T) +{ + return emit_bitcast(ctx, get_current_task_from_pgcstack(ctx.builder, ctx.types().T_size, ctx.pgcstack), T); +} + static Value *get_current_task(jl_codectx_t &ctx) { - return get_current_task_from_pgcstack(ctx.builder, ctx.types().T_size, ctx.pgcstack); + return get_current_task(ctx, ctx.types().T_pjlvalue); } // Get PTLS through current task. @@ -6493,20 +6533,20 @@ static Value *get_current_ptls(jl_codectx_t &ctx) // Get the address of the world age of the current task static Value *get_last_age_field(jl_codectx_t &ctx) { - Value *ct = get_current_task(ctx); + Value *ct = get_current_task(ctx, ctx.types().T_size->getPointerTo()); return ctx.builder.CreateInBoundsGEP( ctx.types().T_size, - ctx.builder.CreateBitCast(ct, ctx.types().T_size->getPointerTo()), + ct, ConstantInt::get(ctx.types().T_size, offsetof(jl_task_t, world_age) / ctx.types().sizeof_ptr), "world_age"); } static Value *get_scope_field(jl_codectx_t &ctx) { - Value *ct = get_current_task(ctx); + Value *ct = get_current_task(ctx, ctx.types().T_prjlvalue->getPointerTo()); return ctx.builder.CreateInBoundsGEP( ctx.types().T_prjlvalue, - ctx.builder.CreateBitCast(ct, ctx.types().T_prjlvalue->getPointerTo()), + ct, ConstantInt::get(ctx.types().T_size, offsetof(jl_task_t, scope) / ctx.types().sizeof_ptr), "current_scope"); } @@ -9119,12 +9159,12 @@ static jl_llvm_functions_t if (lname) { // Save exception stack depth at enter for use in pop_exception Value *excstack_state = - ctx.builder.CreateCall(prepare_call(jl_excstack_state_func)); + ctx.builder.CreateCall(prepare_call(jl_excstack_state_func), {get_current_task(ctx)}); assert(!ctx.ssavalue_assigned[cursor]); ctx.SAvalues[cursor] = jl_cgval_t(excstack_state, (jl_value_t*)jl_ulong_type, NULL); ctx.ssavalue_assigned[cursor] = true; // Actually enter the exception frame - CallInst *sj = ctx.builder.CreateCall(prepare_call(except_enter_func)); + CallInst *sj = ctx.builder.CreateCall(prepare_call(except_enter_func), {get_current_task(ctx)}); // We need to mark this on the call site as well. See issue #6757 sj->setCanReturnTwice(); Value *isz = ctx.builder.CreateICmpEQ(sj, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0)); @@ -9137,8 +9177,7 @@ static jl_llvm_functions_t ctx.builder.CreateCondBr(isz, tryblk, catchpop); ctx.builder.SetInsertPoint(catchpop); { - ctx.builder.CreateCall(prepare_call(jlleave_func), - ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 1)); + ctx.builder.CreateCall(prepare_call(jlleave_func), {get_current_task(ctx), ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 1)}); if (old_scope) { scope_ai.decorateInst( ctx.builder.CreateAlignedStore(old_scope, scope_ptr, ctx.types().alignof_ptr)); @@ -9504,7 +9543,7 @@ jl_llvm_functions_t jl_emit_code( decls.functionObject = ""; decls.specFunctionObject = ""; jl_printf((JL_STREAM*)STDERR_FILENO, "Internal error: encountered unexpected error during compilation of %s:\n", mname.c_str()); - jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception()); + jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception(jl_current_task)); jl_printf((JL_STREAM*)STDERR_FILENO, "\n"); jlbacktrace(); // written to STDERR_FILENO #ifndef JL_NDEBUG @@ -9816,6 +9855,7 @@ static void init_jit_functions(void) add_named_global(jlgenericfunction_func, &jl_generic_function_def); add_named_global(jlenter_func, &jl_enter_handler); add_named_global(jl_current_exception_func, &jl_current_exception); + add_named_global(jlleave_noexcept_func, &jl_pop_handler_noexcept); add_named_global(jlleave_func, &jl_pop_handler); add_named_global(jl_restore_excstack_func, &jl_restore_excstack); add_named_global(jl_excstack_state_func, &jl_excstack_state); diff --git a/src/gc.c b/src/gc.c index c2d194b8a28bb..d57bf05145e74 100644 --- a/src/gc.c +++ b/src/gc.c @@ -301,7 +301,7 @@ static void run_finalizer(jl_task_t *ct, void *o, void *ff) } JL_CATCH { jl_printf((JL_STREAM*)STDERR_FILENO, "error in running finalizer: "); - jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception()); + jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception(ct)); jl_printf((JL_STREAM*)STDERR_FILENO, "\n"); jlbacktrace(); // written to STDERR_FILENO } diff --git a/src/gf.c b/src/gf.c index fdf61f440908b..93c1c61affd2d 100644 --- a/src/gf.c +++ b/src/gf.c @@ -95,7 +95,7 @@ void jl_call_tracer(tracer_cb callback, jl_value_t *tracee) JL_CATCH { ct->ptls->in_pure_callback = last_in; jl_printf((JL_STREAM*)STDERR_FILENO, "WARNING: tracer callback function threw an error:\n"); - jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception()); + jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception(ct)); jl_printf((JL_STREAM*)STDERR_FILENO, "\n"); jlbacktrace(); // written to STDERR_FILENO } @@ -390,7 +390,7 @@ jl_code_info_t *jl_type_infer(jl_method_instance_t *mi, size_t world, int force) src = (jl_code_info_t*)jl_apply(fargs, 3); } JL_CATCH { - jl_value_t *e = jl_current_exception(); + jl_value_t *e = jl_current_exception(ct); jl_printf((JL_STREAM*)STDERR_FILENO, "Internal error: during type inference of\n"); jl_static_show_func_sig((JL_STREAM*)STDERR_FILENO, (jl_value_t*)mi->specTypes); jl_printf((JL_STREAM*)STDERR_FILENO, "\nEncountered "); diff --git a/src/init.c b/src/init.c index 801f12ec53930..2c1ad618948f8 100644 --- a/src/init.c +++ b/src/init.c @@ -273,7 +273,7 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) JL_NOTSAFEPOINT_ENTER } JL_CATCH { jl_printf((JL_STREAM*)STDERR_FILENO, "\natexit hook threw an error: "); - jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception()); + jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception(ct)); jl_printf((JL_STREAM*)STDERR_FILENO, "\n"); jlbacktrace(); // written to STDERR_FILENO } @@ -317,7 +317,7 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) JL_NOTSAFEPOINT_ENTER assert(item); uv_unref(item->h); jl_printf((JL_STREAM*)STDERR_FILENO, "error during exit cleanup: close: "); - jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception()); + jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception(ct)); jl_printf((JL_STREAM*)STDERR_FILENO, "\n"); jlbacktrace(); // written to STDERR_FILENO item = next_shutdown_queue_item(item); @@ -372,7 +372,7 @@ JL_DLLEXPORT void jl_postoutput_hook(void) } JL_CATCH { jl_printf((JL_STREAM*)STDERR_FILENO, "\npostoutput hook threw an error: "); - jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception()); + jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception(ct)); jl_printf((JL_STREAM*)STDERR_FILENO, "\n"); jlbacktrace(); // written to STDERR_FILENO } diff --git a/src/interpreter.c b/src/interpreter.c index 5102d1417c939..2fb4b91927496 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -317,7 +317,7 @@ static jl_value_t *eval_value(jl_value_t *e, interpreter_state *s) return jl_copy_ast(eval_value(args[0], s)); } else if (head == jl_exc_sym) { - return jl_current_exception(); + return jl_current_exception(jl_current_task); } else if (head == jl_boundscheck_sym) { return jl_true; @@ -490,7 +490,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, s->locals[jl_source_nslots(s->src) + id] = val; } else if (jl_is_enternode(stmt)) { - jl_enter_handler(&__eh); + jl_enter_handler(ct, &__eh); // This is a bit tricky, but supports the implementation of PhiC nodes. // They are conceptually slots, but the slot to store to doesn't get explicitly // mentioned in the store (aka the "UpsilonNode") (this makes them integrate more @@ -521,7 +521,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, } // store current top of exception stack for restore in pop_exception. } - s->locals[jl_source_nslots(s->src) + ip] = jl_box_ulong(jl_excstack_state()); + s->locals[jl_source_nslots(s->src) + ip] = jl_box_ulong(jl_excstack_state(ct)); if (jl_enternode_scope(stmt)) { jl_value_t *old_scope = ct->scope; JL_GC_PUSH1(&old_scope); @@ -540,13 +540,15 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, jl_unreachable(); } } - jl_eh_restore_state(&__eh); + if (s->continue_at) { // means we reached a :leave expression + jl_eh_restore_state_noexcept(ct, &__eh); ip = s->continue_at; s->continue_at = 0; continue; } else { // a real exception + jl_eh_restore_state(ct, &__eh); ip = catch_ip; assert(jl_enternode_catch_dest(stmt) != 0); continue; @@ -609,7 +611,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, } else if (head == jl_pop_exception_sym) { size_t prev_state = jl_unbox_ulong(eval_value(jl_exprarg(stmt, 0), s)); - jl_restore_excstack(prev_state); + jl_restore_excstack(ct, prev_state); } else if (toplevel) { if (head == jl_method_sym && jl_expr_nargs(stmt) > 1) { diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index de1bc7601f60d..db23bcbc33f6c 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -109,6 +109,7 @@ XX(jl_egal__bits) \ XX(jl_egal__bitstag) \ XX(jl_eh_restore_state) \ + XX(jl_eh_restore_state_noexcept) \ XX(jl_enter_handler) \ XX(jl_enter_threaded_region) \ XX(jl_environ) \ @@ -367,6 +368,7 @@ XX(jl_pointerref) \ XX(jl_pointerset) \ XX(jl_pop_handler) \ + XX(jl_pop_handler_noexcept) \ XX(jl_preload_sysimg_so) \ XX(jl_prepend_cwd) \ XX(jl_printf) \ diff --git a/src/jlapi.c b/src/jlapi.c index 276a792b2cd66..a3621385a437e 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -158,7 +158,7 @@ JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str) _jl_exception_clear(ct); } JL_CATCH { - ct->ptls->previous_exception = jl_current_exception(); + ct->ptls->previous_exception = jl_current_exception(ct); r = NULL; } return r; @@ -170,9 +170,9 @@ JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str) * @return A pointer to `jl_value_t` representing the current exception. * Returns `NULL` if no exception is currently thrown. */ -JL_DLLEXPORT jl_value_t *jl_current_exception(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT +JL_DLLEXPORT jl_value_t *jl_current_exception(jl_task_t *ct) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT { - jl_excstack_t *s = jl_current_task->excstack; + jl_excstack_t *s = ct->excstack; return s && s->top != 0 ? jl_excstack_exception(s, s->top) : jl_nothing; } @@ -300,7 +300,7 @@ JL_DLLEXPORT jl_value_t *jl_call(jl_function_t *f, jl_value_t **args, uint32_t n _jl_exception_clear(ct); } JL_CATCH { - ct->ptls->previous_exception = jl_current_exception(); + ct->ptls->previous_exception = jl_current_exception(ct); v = NULL; } return v; @@ -328,7 +328,7 @@ JL_DLLEXPORT jl_value_t *jl_call0(jl_function_t *f) _jl_exception_clear(ct); } JL_CATCH { - ct->ptls->previous_exception = jl_current_exception(); + ct->ptls->previous_exception = jl_current_exception(ct); v = NULL; } return v; @@ -360,7 +360,7 @@ JL_DLLEXPORT jl_value_t *jl_call1(jl_function_t *f, jl_value_t *a) _jl_exception_clear(ct); } JL_CATCH { - ct->ptls->previous_exception = jl_current_exception(); + ct->ptls->previous_exception = jl_current_exception(ct); v = NULL; } return v; @@ -394,7 +394,7 @@ JL_DLLEXPORT jl_value_t *jl_call2(jl_function_t *f, jl_value_t *a, jl_value_t *b _jl_exception_clear(ct); } JL_CATCH { - ct->ptls->previous_exception = jl_current_exception(); + ct->ptls->previous_exception = jl_current_exception(ct); v = NULL; } return v; @@ -431,7 +431,7 @@ JL_DLLEXPORT jl_value_t *jl_call3(jl_function_t *f, jl_value_t *a, _jl_exception_clear(ct); } JL_CATCH { - ct->ptls->previous_exception = jl_current_exception(); + ct->ptls->previous_exception = jl_current_exception(ct); v = NULL; } return v; @@ -447,6 +447,7 @@ JL_DLLEXPORT jl_value_t *jl_call3(jl_function_t *f, jl_value_t *a, JL_DLLEXPORT jl_value_t *jl_get_field(jl_value_t *o, const char *fld) { jl_value_t *v; + jl_task_t *ct = jl_current_task; JL_TRY { jl_value_t *s = (jl_value_t*)jl_symbol(fld); int i = jl_field_index((jl_datatype_t*)jl_typeof(o), (jl_sym_t*)s, 1); @@ -454,7 +455,7 @@ JL_DLLEXPORT jl_value_t *jl_get_field(jl_value_t *o, const char *fld) jl_exception_clear(); } JL_CATCH { - jl_current_task->ptls->previous_exception = jl_current_exception(); + ct->ptls->previous_exception = jl_current_exception(ct); v = NULL; } return v; @@ -852,6 +853,7 @@ JL_DLLEXPORT int jl_set_fenv_rounding(int i) static int exec_program(char *program) { + jl_task_t *ct = jl_current_task; JL_TRY { jl_load(jl_main_module, program); } @@ -860,7 +862,7 @@ static int exec_program(char *program) // printing directly to STDERR_FILENO. int shown_err = 0; jl_printf(JL_STDERR, "error during bootstrap:\n"); - jl_value_t *exc = jl_current_exception(); + jl_value_t *exc = jl_current_exception(ct); jl_value_t *showf = jl_base_module ? jl_get_function(jl_base_module, "show") : NULL; if (showf) { jl_value_t *errs = jl_stderr_obj(); @@ -889,8 +891,8 @@ static NOINLINE int true_main(int argc, char *argv[]) jl_function_t *start_client = jl_base_module ? (jl_function_t*)jl_get_global(jl_base_module, jl_symbol("_start")) : NULL; + jl_task_t *ct = jl_current_task; if (start_client) { - jl_task_t *ct = jl_current_task; int ret = 1; JL_TRY { size_t last_age = ct->world_age; @@ -902,7 +904,7 @@ static NOINLINE int true_main(int argc, char *argv[]) ct->world_age = last_age; } JL_CATCH { - jl_no_exc_handler(jl_current_exception(), ct); + jl_no_exc_handler(jl_current_exception(ct), ct); } return ret; } @@ -946,7 +948,7 @@ static NOINLINE int true_main(int argc, char *argv[]) line = NULL; } jl_printf((JL_STREAM*)STDERR_FILENO, "\nparser error:\n"); - jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception()); + jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception(ct)); jl_printf((JL_STREAM*)STDERR_FILENO, "\n"); jl_print_backtrace(); // written to STDERR_FILENO } diff --git a/src/jloptions.c b/src/jloptions.c index 570d021351104..c26c4e26cedd7 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -851,7 +851,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) if (isnan(sz) || sz < 0) { jl_errorf("julia: invalid argument to --heap-size-hint (%s)", optarg); } - jl_options.heap_size_hint = sz < UINT64_MAX ? (uint64_t)sz : UINT64_MAX; + const long double limit = ldexpl(1.0, 64); // UINT64_MAX + 1 + jl_options.heap_size_hint = sz < limit ? (uint64_t)sz : UINT64_MAX; } if (jl_options.heap_size_hint == 0) jl_errorf("julia: invalid memory size specified in --heap-size-hint"); diff --git a/src/jltypes.c b/src/jltypes.c index ef58deef2c634..9d238dc27b010 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1846,7 +1846,7 @@ static jl_value_t *normalize_unionalls(jl_value_t *t) else if (jl_is_unionall(t)) { jl_unionall_t *u = (jl_unionall_t*)t; jl_value_t *body = normalize_unionalls(u->body); - JL_GC_PUSH1(&body); + JL_GC_PUSH2(&body, &t); if (body != u->body) { t = jl_new_struct(jl_unionall_type, u->var, body); u = (jl_unionall_t*)t; diff --git a/src/julia.h b/src/julia.h index 6259da6416f3d..602d7e5c4318d 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2004,17 +2004,6 @@ JL_DLLEXPORT void JL_NORETURN jl_bounds_error_unboxed_int(void *v, jl_value_t *v JL_DLLEXPORT void JL_NORETURN jl_bounds_error_ints(jl_value_t *v JL_MAYBE_UNROOTED, size_t *idxs, size_t nidxs); -// Return the exception currently being handled, or `jl_nothing`. -// -// The catch scope is determined dynamically so this works in functions called -// from a catch block. The returned value is gc rooted until we exit the -// enclosing JL_CATCH. -// FIXME: Teach the static analyzer about this rather than using -// JL_GLOBALLY_ROOTED which is far too optimistic. -JL_DLLEXPORT jl_value_t *jl_current_exception(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_value_t *jl_exception_occurred(void); -JL_DLLEXPORT void jl_exception_clear(void) JL_NOTSAFEPOINT; - #define JL_NARGS(fname, min, max) \ if (nargs < min) jl_too_few_args(#fname, min); \ else if (nargs > max) jl_too_many_args(#fname, max); @@ -2273,11 +2262,24 @@ extern JL_DLLIMPORT int jl_task_ptls_offset; #include "julia_locks.h" // requires jl_task_t definition -JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh); -JL_DLLEXPORT void jl_eh_restore_state(jl_handler_t *eh); -JL_DLLEXPORT void jl_pop_handler(int n); -JL_DLLEXPORT size_t jl_excstack_state(void) JL_NOTSAFEPOINT; -JL_DLLEXPORT void jl_restore_excstack(size_t state) JL_NOTSAFEPOINT; +// Return the exception currently being handled, or `jl_nothing`. +// +// The catch scope is determined dynamically so this works in functions called +// from a catch block. The returned value is gc rooted until we exit the +// enclosing JL_CATCH. +// FIXME: Teach the static analyzer about this rather than using +// JL_GLOBALLY_ROOTED which is far too optimistic. +JL_DLLEXPORT jl_value_t *jl_current_exception(jl_task_t *ct) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_exception_occurred(void); +JL_DLLEXPORT void jl_exception_clear(void) JL_NOTSAFEPOINT; + +JL_DLLEXPORT void jl_enter_handler(jl_task_t *ct, jl_handler_t *eh) JL_NOTSAFEPOINT ; +JL_DLLEXPORT void jl_eh_restore_state(jl_task_t *ct, jl_handler_t *eh); +JL_DLLEXPORT void jl_eh_restore_state_noexcept(jl_task_t *ct, jl_handler_t *eh) JL_NOTSAFEPOINT; +JL_DLLEXPORT void jl_pop_handler(jl_task_t *ct, int n) JL_NOTSAFEPOINT; +JL_DLLEXPORT void jl_pop_handler_noexcept(jl_task_t *ct, int n) JL_NOTSAFEPOINT; +JL_DLLEXPORT size_t jl_excstack_state(jl_task_t *ct) JL_NOTSAFEPOINT; +JL_DLLEXPORT void jl_restore_excstack(jl_task_t *ct, size_t state) JL_NOTSAFEPOINT; #if defined(_OS_WINDOWS_) #if defined(_COMPILER_GCC_) @@ -2328,24 +2330,36 @@ extern void (*real_siglongjmp)(jmp_buf _Buf, int _Value); #ifdef __clang_gcanalyzer__ -// This is hard. Ideally we'd teach the static analyzer about the extra control -// flow edges. But for now, just hide this as best we can extern int had_exception; -#define JL_TRY if (1) -#define JL_CATCH if (had_exception) -#else +// The analyzer assumes that the TRY block always executes to completion. +// This can lead to both false positives and false negatives, since it doesn't model the fact that throwing always leaves the try block early. +#define JL_TRY \ + int i__try, i__catch; jl_handler_t __eh; jl_task_t *__eh_ct; \ + __eh_ct = jl_current_task; \ + size_t __excstack_state = jl_excstack_state(__eh_ct); \ + jl_enter_handler(__eh_ct, &__eh); \ + if (1) + /* TRY BLOCK; */ +#define JL_CATCH \ + if (!had_exception) \ + jl_eh_restore_state_noexcept(__eh_ct, &__eh); \ + else \ + for (i__catch=1, jl_eh_restore_state(__eh_ct, &__eh); i__catch; i__catch=0, /* CATCH BLOCK; */ jl_restore_excstack(__eh_ct, __excstack_state)) -#define JL_TRY \ - int i__tr, i__ca; jl_handler_t __eh; \ - size_t __excstack_state = jl_excstack_state(); \ - jl_enter_handler(&__eh); \ - if (!jl_setjmp(__eh.eh_ctx,0)) \ - for (i__tr=1; i__tr; i__tr=0, jl_eh_restore_state(&__eh)) +#else -#define JL_CATCH \ - else \ - for (i__ca=1, jl_eh_restore_state(&__eh); i__ca; i__ca=0, jl_restore_excstack(__excstack_state)) +#define JL_TRY \ + int i__try, i__catch; jl_handler_t __eh; jl_task_t *__eh_ct; \ + __eh_ct = jl_current_task; \ + size_t __excstack_state = jl_excstack_state(__eh_ct); \ + jl_enter_handler(__eh_ct, &__eh); \ + if (!jl_setjmp(__eh.eh_ctx, 0)) \ + for (i__try=1; i__try; i__try=0, /* TRY BLOCK; */ jl_eh_restore_state_noexcept(__eh_ct, &__eh)) + +#define JL_CATCH \ + else \ + for (i__catch=1, jl_eh_restore_state(__eh_ct, &__eh); i__catch; i__catch=0, /* CATCH BLOCK; */ jl_restore_excstack(__eh_ct, __excstack_state)) #endif diff --git a/src/julia_internal.h b/src/julia_internal.h index 2dc8fd0c105b6..cdb543e440c2d 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1268,7 +1268,7 @@ STATIC_INLINE size_t jl_excstack_next(jl_excstack_t *stack, size_t itr) JL_NOTSA return itr-2 - jl_excstack_bt_size(stack, itr); } // Exception stack manipulation -void jl_push_excstack(jl_task_t* task, jl_excstack_t **stack JL_REQUIRE_ROOTED_SLOT JL_ROOTING_ARGUMENT, +void jl_push_excstack(jl_task_t *ct, jl_excstack_t **stack JL_REQUIRE_ROOTED_SLOT JL_ROOTING_ARGUMENT, jl_value_t *exception JL_ROOTED_ARGUMENT, jl_bt_element_t *bt_data, size_t bt_size); diff --git a/src/llvm-codegen-shared.h b/src/llvm-codegen-shared.h index 2c60907ca086c..b355dd05436c1 100644 --- a/src/llvm-codegen-shared.h +++ b/src/llvm-codegen-shared.h @@ -222,7 +222,7 @@ static inline llvm::Value *get_current_ptls_from_task(llvm::IRBuilder<> &builder auto T_pjlvalue = JuliaType::get_pjlvalue_ty(builder.getContext()); const int ptls_offset = offsetof(jl_task_t, ptls); llvm::Value *pptls = builder.CreateInBoundsGEP( - T_pjlvalue, current_task, + T_pjlvalue, emit_bitcast_with_builder(builder, current_task, T_ppjlvalue), ConstantInt::get(T_size, ptls_offset / sizeof(void *)), "ptls_field"); LoadInst *ptls_load = builder.CreateAlignedLoad(T_pjlvalue, diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index e081c6f067245..0fb5b9bb18805 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -1621,7 +1621,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { callee == pgcstack_getter || callee->getName() == XSTR(jl_egal__unboxed) || callee->getName() == XSTR(jl_lock_value) || callee->getName() == XSTR(jl_unlock_value) || callee->getName() == XSTR(jl_lock_field) || callee->getName() == XSTR(jl_unlock_field) || - callee == write_barrier_func || callee == gc_loaded_func || + callee == write_barrier_func || callee == gc_loaded_func || callee == pop_handler_noexcept_func || callee->getName() == "memcmp") { continue; } diff --git a/src/llvm-lower-handlers.cpp b/src/llvm-lower-handlers.cpp index 15866d0855fc1..d6d4793f3c1c0 100644 --- a/src/llvm-lower-handlers.cpp +++ b/src/llvm-lower-handlers.cpp @@ -37,13 +37,13 @@ using namespace llvm; /* Lowers Julia Exception Handlers and colors EH frames. * * Our task is to lower: - * call void @julia.except_enter() + * call void @julia.except_enter(ct) * <...> * call void jl_pop_handler(1) * * to * - * call void @jl_enter_handler(jl_handler *%buff) + * call void @jl_enter_handler(ct, jl_handler *%buff) * <...> * call void jl_pop_handler(1) * @@ -81,25 +81,19 @@ namespace { * If the module doesn't have declarations for the jl_enter_handler and setjmp * functions, insert them. */ -static void ensure_enter_function(Module &M, const Triple &TT) +static void ensure_enter_function(Module &M, Type *T_pjlvalue, const Triple &TT) { auto T_int8 = Type::getInt8Ty(M.getContext()); auto T_pint8 = PointerType::get(T_int8, 0); auto T_void = Type::getVoidTy(M.getContext()); auto T_int32 = Type::getInt32Ty(M.getContext()); if (!M.getNamedValue(XSTR(jl_enter_handler))) { - SmallVector ehargs(0); - ehargs.push_back(T_pint8); - Function::Create(FunctionType::get(T_void, ehargs, false), + Function::Create(FunctionType::get(T_void, {T_pjlvalue, T_pint8}, false), Function::ExternalLinkage, XSTR(jl_enter_handler), &M); } if (!M.getNamedValue(jl_setjmp_name)) { - SmallVector args2(0); - args2.push_back(T_pint8); - if (!TT.isOSWindows()) { - args2.push_back(T_int32); - } - Function::Create(FunctionType::get(T_int32, args2, false), + Type *args2[] = {T_pint8, T_int32}; + Function::Create(FunctionType::get(T_int32, ArrayRef(args2, TT.isOSWindows() ? 1 : 2), false), Function::ExternalLinkage, jl_setjmp_name, &M) ->addFnAttr(Attribute::ReturnsTwice); } @@ -111,8 +105,9 @@ static bool lowerExcHandlers(Function &F) { Function *except_enter_func = M.getFunction("julia.except_enter"); if (!except_enter_func) return false; // No EH frames in this module - ensure_enter_function(M, TT); + ensure_enter_function(M, except_enter_func->getFunctionType()->getParamType(0), TT); Function *leave_func = M.getFunction(XSTR(jl_pop_handler)); + Function *leave_noexcept_func = M.getFunction(XSTR(jl_pop_handler_noexcept)); Function *jlenter_func = M.getFunction(XSTR(jl_enter_handler)); Function *setjmp_func = M.getFunction(jl_setjmp_name); @@ -150,9 +145,9 @@ static bool lowerExcHandlers(Function &F) { continue; if (Callee == except_enter_func) EnterDepth[CI] = Depth++; - else if (Callee == leave_func) { + else if (Callee == leave_func || Callee == leave_noexcept_func) { LeaveDepth[CI] = Depth; - Depth -= cast(CI->getArgOperand(0))->getLimitedValue(); + Depth -= cast(CI->getArgOperand(1))->getLimitedValue(); } assert(Depth >= 0); if (Depth > MaxDepth) @@ -192,7 +187,7 @@ static bool lowerExcHandlers(Function &F) { assert(it.second >= 0); Instruction *buff = buffs[it.second]; CallInst *enter = it.first; - auto new_enter = CallInst::Create(jlenter_func, buff, "", enter); + auto new_enter = CallInst::Create(jlenter_func, {enter->getArgOperand(0), buff}, "", enter); Value *lifetime_args[] = { handler_sz64, buff @@ -200,10 +195,7 @@ static bool lowerExcHandlers(Function &F) { CallInst::Create(lifetime_start, lifetime_args, "", new_enter); CallInst *sj; if (!TT.isOSWindows()) { - // For LLVM 3.3 compatibility - Value *args[] = {buff, - ConstantInt::get(Type::getInt32Ty(F.getContext()), 0)}; - sj = CallInst::Create(setjmp_func, args, "", enter); + sj = CallInst::Create(setjmp_func, {buff, ConstantInt::get(Type::getInt32Ty(F.getContext()), 0)}, "", enter); } else { sj = CallInst::Create(setjmp_func, buff, "", enter); } @@ -219,7 +211,7 @@ static bool lowerExcHandlers(Function &F) { // Insert lifetime end intrinsics after every leave. for (auto it : LeaveDepth) { int StartDepth = it.second - 1; - int npops = cast(it.first->getArgOperand(0))->getLimitedValue(); + int npops = cast(it.first->getArgOperand(1))->getLimitedValue(); for (int i = 0; i < npops; ++i) { assert(StartDepth-i >= 0); Value *lifetime_args[] = { diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index 9c62a8a5711a5..f217d27035200 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -27,7 +27,7 @@ JuliaPassContext::JuliaPassContext() pgcstack_getter(nullptr), adoptthread_func(nullptr), gc_flush_func(nullptr), gc_preserve_begin_func(nullptr), gc_preserve_end_func(nullptr), pointer_from_objref_func(nullptr), gc_loaded_func(nullptr), alloc_obj_func(nullptr), - typeof_func(nullptr), write_barrier_func(nullptr), + typeof_func(nullptr), write_barrier_func(nullptr), pop_handler_noexcept_func(nullptr), call_func(nullptr), call2_func(nullptr), call3_func(nullptr), module(nullptr) { } @@ -53,6 +53,7 @@ void JuliaPassContext::initFunctions(Module &M) typeof_func = M.getFunction("julia.typeof"); write_barrier_func = M.getFunction("julia.write_barrier"); alloc_obj_func = M.getFunction("julia.gc_alloc_obj"); + pop_handler_noexcept_func = M.getFunction(XSTR(jl_pop_handler_noexcept)); call_func = M.getFunction("julia.call"); call2_func = M.getFunction("julia.call2"); call3_func = M.getFunction("julia.call3"); diff --git a/src/llvm-pass-helpers.h b/src/llvm-pass-helpers.h index 346500df51ca1..d0ac7faa01d2c 100644 --- a/src/llvm-pass-helpers.h +++ b/src/llvm-pass-helpers.h @@ -60,6 +60,7 @@ struct JuliaPassContext { llvm::Function *alloc_obj_func; llvm::Function *typeof_func; llvm::Function *write_barrier_func; + llvm::Function *pop_handler_noexcept_func; llvm::Function *call_func; llvm::Function *call2_func; llvm::Function *call3_func; diff --git a/src/method.c b/src/method.c index 88d43fb4b22ee..ac5475d54a4d5 100644 --- a/src/method.c +++ b/src/method.c @@ -129,6 +129,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve } if (e->head == jl_cfunction_sym) { JL_NARGS(cfunction method definition, 5, 5); // (type, func, rt, at, cc) + jl_task_t *ct = jl_current_task; jl_value_t *typ = jl_exprarg(e, 0); if (!jl_is_type(typ)) jl_error("first parameter to :cfunction must be a type"); @@ -145,7 +146,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve rt = jl_interpret_toplevel_expr_in(module, rt, NULL, sparam_vals); } JL_CATCH { - if (jl_typetagis(jl_current_exception(), jl_errorexception_type)) + if (jl_typetagis(jl_current_exception(ct), jl_errorexception_type)) jl_error("could not evaluate cfunction return type (it might depend on a local variable)"); else jl_rethrow(); @@ -157,7 +158,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve at = jl_interpret_toplevel_expr_in(module, at, NULL, sparam_vals); } JL_CATCH { - if (jl_typetagis(jl_current_exception(), jl_errorexception_type)) + if (jl_typetagis(jl_current_exception(ct), jl_errorexception_type)) jl_error("could not evaluate cfunction argument type (it might depend on a local variable)"); else jl_rethrow(); @@ -171,6 +172,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve } if (e->head == jl_foreigncall_sym) { JL_NARGSV(ccall method definition, 5); // (fptr, rt, at, nreq, (cc, effects)) + jl_task_t *ct = jl_current_task; jl_value_t *rt = jl_exprarg(e, 1); jl_value_t *at = jl_exprarg(e, 2); if (!jl_is_type(rt)) { @@ -178,7 +180,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve rt = jl_interpret_toplevel_expr_in(module, rt, NULL, sparam_vals); } JL_CATCH { - if (jl_typetagis(jl_current_exception(), jl_errorexception_type)) + if (jl_typetagis(jl_current_exception(ct), jl_errorexception_type)) jl_error("could not evaluate ccall return type (it might depend on a local variable)"); else jl_rethrow(); @@ -190,7 +192,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve at = jl_interpret_toplevel_expr_in(module, at, NULL, sparam_vals); } JL_CATCH { - if (jl_typetagis(jl_current_exception(), jl_errorexception_type)) + if (jl_typetagis(jl_current_exception(ct), jl_errorexception_type)) jl_error("could not evaluate ccall argument type (it might depend on a local variable)"); else jl_rethrow(); @@ -266,6 +268,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve val = jl_interpret_toplevel_expr_in(module, (jl_value_t*)e, NULL, sparam_vals); } JL_CATCH { + val = NULL; // To make the analyzer happy see #define JL_TRY } if (val) return val; diff --git a/src/rtutils.c b/src/rtutils.c index 1ab881e613564..a1c1622b0b5ee 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -239,9 +239,8 @@ JL_DLLEXPORT void __stack_chk_fail(void) // exceptions ----------------------------------------------------------------- -JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh) +JL_DLLEXPORT void jl_enter_handler(jl_task_t *ct, jl_handler_t *eh) { - jl_task_t *ct = jl_current_task; // Must have no safepoint eh->prev = ct->eh; eh->gcstack = ct->gcstack; @@ -260,9 +259,8 @@ JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh) // * We leave a try block through normal control flow // * An exception causes a nonlocal jump to the catch block. In this case // there's additional cleanup required, eg pushing the exception stack. -JL_DLLEXPORT void jl_eh_restore_state(jl_handler_t *eh) +JL_DLLEXPORT void jl_eh_restore_state(jl_task_t *ct, jl_handler_t *eh) { - jl_task_t *ct = jl_current_task; #ifdef _OS_WINDOWS_ if (ct->ptls->needs_resetstkoflw) { _resetstkoflw(); @@ -297,27 +295,40 @@ JL_DLLEXPORT void jl_eh_restore_state(jl_handler_t *eh) } } -JL_DLLEXPORT void jl_pop_handler(int n) +JL_DLLEXPORT void jl_eh_restore_state_noexcept(jl_task_t *ct, jl_handler_t *eh) +{ + ct->eh = eh->prev; + ct->ptls->defer_signal = eh->defer_signal; // optional, but certain try-finally (in stream.jl) may be slightly harder to write without this +} + +JL_DLLEXPORT void jl_pop_handler(jl_task_t *ct, int n) +{ + if (__unlikely(n <= 0)) + return; + jl_handler_t *eh = ct->eh; + while (--n > 0) + eh = eh->prev; + jl_eh_restore_state(ct, eh); +} + +JL_DLLEXPORT void jl_pop_handler_noexcept(jl_task_t *ct, int n) { - jl_task_t *ct = jl_current_task; if (__unlikely(n <= 0)) return; jl_handler_t *eh = ct->eh; while (--n > 0) eh = eh->prev; - jl_eh_restore_state(eh); + jl_eh_restore_state_noexcept(ct, eh); } -JL_DLLEXPORT size_t jl_excstack_state(void) JL_NOTSAFEPOINT +JL_DLLEXPORT size_t jl_excstack_state(jl_task_t *ct) JL_NOTSAFEPOINT { - jl_task_t *ct = jl_current_task; jl_excstack_t *s = ct->excstack; return s ? s->top : 0; } -JL_DLLEXPORT void jl_restore_excstack(size_t state) JL_NOTSAFEPOINT +JL_DLLEXPORT void jl_restore_excstack(jl_task_t *ct, size_t state) JL_NOTSAFEPOINT { - jl_task_t *ct = jl_current_task; jl_excstack_t *s = ct->excstack; if (s) { assert(s->top >= state); @@ -332,28 +343,27 @@ static void jl_copy_excstack(jl_excstack_t *dest, jl_excstack_t *src) JL_NOTSAFE dest->top = src->top; } -static void jl_reserve_excstack(jl_task_t* task, jl_excstack_t **stack JL_REQUIRE_ROOTED_SLOT, +static void jl_reserve_excstack(jl_task_t *ct, jl_excstack_t **stack JL_REQUIRE_ROOTED_SLOT, size_t reserved_size) { jl_excstack_t *s = *stack; if (s && s->reserved_size >= reserved_size) return; size_t bufsz = sizeof(jl_excstack_t) + sizeof(uintptr_t)*reserved_size; - jl_task_t *ct = jl_current_task; jl_excstack_t *new_s = (jl_excstack_t*)jl_gc_alloc_buf(ct->ptls, bufsz); new_s->top = 0; new_s->reserved_size = reserved_size; if (s) jl_copy_excstack(new_s, s); *stack = new_s; - jl_gc_wb(task, new_s); + jl_gc_wb(ct, new_s); } -void jl_push_excstack(jl_task_t* task, jl_excstack_t **stack JL_REQUIRE_ROOTED_SLOT JL_ROOTING_ARGUMENT, +void jl_push_excstack(jl_task_t *ct, jl_excstack_t **stack JL_REQUIRE_ROOTED_SLOT JL_ROOTING_ARGUMENT, jl_value_t *exception JL_ROOTED_ARGUMENT, jl_bt_element_t *bt_data, size_t bt_size) { - jl_reserve_excstack(task, stack, (*stack ? (*stack)->top : 0) + bt_size + 2); + jl_reserve_excstack(ct, stack, (*stack ? (*stack)->top : 0) + bt_size + 2); jl_excstack_t *s = *stack; jl_bt_element_t *rawstack = jl_excstack_raw(s); memcpy(rawstack + s->top, bt_data, sizeof(jl_bt_element_t)*bt_size); diff --git a/src/scheduler.c b/src/scheduler.c index dbc913353ce4c..8b778b877e025 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -110,7 +110,7 @@ void jl_init_threadinginfra(void) } -void JL_NORETURN jl_finish_task(jl_task_t *t); +void JL_NORETURN jl_finish_task(jl_task_t *ct); static inline int may_mark(void) JL_NOTSAFEPOINT { diff --git a/src/signals-mach.c b/src/signals-mach.c index 5063aa3e8e0d2..56f7c0d7505f4 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -115,7 +115,7 @@ static void jl_mach_gc_wait(jl_ptls_t ptls2, mach_port_t thread, int16_t tid) // Eventually, we should probably release this signal to the original // thread, (return KERN_FAILURE instead of KERN_SUCCESS) so that it // triggers a SIGSEGV and gets handled by the usual codepath for unix. - int8_t gc_state = ptls2->gc_state; + int8_t gc_state = jl_atomic_load_acquire(&ptls2->gc_state); jl_atomic_store_release(&ptls2->gc_state, JL_GC_STATE_WAITING); uintptr_t item = tid | (((uintptr_t)gc_state) << 16); arraylist_push(&suspended_threads, (void*)item); @@ -338,7 +338,7 @@ kern_return_t catch_mach_exception_raise( // jl_throw_in_thread(ptls2, thread, jl_stackovf_exception); // return KERN_SUCCESS; // } - if (ptls2->gc_state == JL_GC_STATE_WAITING) + if (jl_atomic_load_acquire(&ptls2->gc_state) == JL_GC_STATE_WAITING) return KERN_FAILURE; if (exception == EXC_ARITHMETIC) { jl_throw_in_thread(ptls2, thread, jl_diverror_exception); @@ -363,7 +363,7 @@ kern_return_t catch_mach_exception_raise( } return KERN_SUCCESS; } - if (ptls2->current_task->eh == NULL) + if (jl_atomic_load_relaxed(&ptls2->current_task)->eh == NULL) return KERN_FAILURE; jl_value_t *excpt; if (is_addr_on_stack(jl_atomic_load_relaxed(&ptls2->current_task), (void*)fault_addr)) { diff --git a/src/task.c b/src/task.c index 34e68e738abae..d6563888c1663 100644 --- a/src/task.c +++ b/src/task.c @@ -290,18 +290,17 @@ JL_NO_ASAN static void restore_stack2(jl_task_t *t, jl_ptls_t ptls, jl_task_t *l /* Rooted by the base module */ static _Atomic(jl_function_t*) task_done_hook_func JL_GLOBALLY_ROOTED = NULL; -void JL_NORETURN jl_finish_task(jl_task_t *t) +void JL_NORETURN jl_finish_task(jl_task_t *ct) { - jl_task_t *ct = jl_current_task; JL_PROBE_RT_FINISH_TASK(ct); JL_SIGATOMIC_BEGIN(); - if (jl_atomic_load_relaxed(&t->_isexception)) - jl_atomic_store_release(&t->_state, JL_TASK_STATE_FAILED); + if (jl_atomic_load_relaxed(&ct->_isexception)) + jl_atomic_store_release(&ct->_state, JL_TASK_STATE_FAILED); else - jl_atomic_store_release(&t->_state, JL_TASK_STATE_DONE); - if (t->copy_stack) { // early free of stkbuf - asan_free_copy_stack(t->stkbuf, t->bufsz); - t->stkbuf = NULL; + jl_atomic_store_release(&ct->_state, JL_TASK_STATE_DONE); + if (ct->copy_stack) { // early free of stkbuf + asan_free_copy_stack(ct->stkbuf, ct->bufsz); + ct->stkbuf = NULL; } // ensure that state is cleared ct->ptls->in_finalizer = 0; @@ -315,12 +314,12 @@ void JL_NORETURN jl_finish_task(jl_task_t *t) jl_atomic_store_release(&task_done_hook_func, done); } if (done != NULL) { - jl_value_t *args[2] = {done, (jl_value_t*)t}; + jl_value_t *args[2] = {done, (jl_value_t*)ct}; JL_TRY { jl_apply(args, 2); } JL_CATCH { - jl_no_exc_handler(jl_current_exception(), ct); + jl_no_exc_handler(jl_current_exception(ct), ct); } } jl_gc_debug_critical_error(); @@ -688,7 +687,7 @@ JL_DLLEXPORT JL_NORETURN void jl_no_exc_handler(jl_value_t *e, jl_task_t *ct) // NULL exception objects are used when rethrowing. we don't have a handler to process // the exception stack, so at least report the exception at the top of the stack. if (!e) - e = jl_current_exception(); + e = jl_current_exception(ct); jl_printf((JL_STREAM*)STDERR_FILENO, "fatal: error thrown and no exception handler available.\n"); jl_static_show((JL_STREAM*)STDERR_FILENO, e); @@ -721,8 +720,8 @@ JL_DLLEXPORT JL_NORETURN void jl_no_exc_handler(jl_value_t *e, jl_task_t *ct) /* The temporary ptls->bt_data is rooted by special purpose code in the\ GC. This exists only for the purpose of preserving bt_data until we \ set ptls->bt_size=0 below. */ \ - jl_push_excstack(ct, &ct->excstack, exception, \ - ptls->bt_data, ptls->bt_size); \ + jl_push_excstack(ct, &ct->excstack, exception, \ + ptls->bt_data, ptls->bt_size); \ ptls->bt_size = 0; \ } \ assert(ct->excstack && ct->excstack->top); \ @@ -1203,7 +1202,7 @@ CFI_NORETURN res = jl_apply(&ct->start, 1); } JL_CATCH { - res = jl_current_exception(); + res = jl_current_exception(ct); jl_atomic_store_relaxed(&ct->_isexception, 1); goto skip_pop_exception; } diff --git a/src/toplevel.c b/src/toplevel.c index e4438da991e1e..caa01015c362a 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -82,7 +82,7 @@ void jl_module_run_initializer(jl_module_t *m) } else { jl_rethrow_other(jl_new_struct(jl_initerror_type, m->name, - jl_current_exception())); + jl_current_exception(ct))); } } } @@ -1081,7 +1081,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, jl_rethrow(); else jl_rethrow_other(jl_new_struct(jl_loaderror_type, filename, result, - jl_current_exception())); + jl_current_exception(ct))); } JL_GC_POP(); return result; diff --git a/test/Makefile b/test/Makefile index e522ec811b167..31f8f20615d7a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -70,5 +70,6 @@ clangsa: clean: @$(MAKE) -C embedding $@ $(EMBEDDING_ARGS) @$(MAKE) -C gcext $@ $(GCEXT_ARGS) + @$(MAKE) -C llvmpasses $@ .PHONY: $(TESTS) $(addprefix revise-, $(TESTS)) relocatedepot revise-relocatedepot embedding gcext clangsa clean diff --git a/test/llvmpasses/Makefile b/test/llvmpasses/Makefile index 7318d1b67da02..d9fdfa190f3cf 100644 --- a/test/llvmpasses/Makefile +++ b/test/llvmpasses/Makefile @@ -30,4 +30,7 @@ update-help: $(JULIAHOME)/deps/srccache/llvm/llvm/utils/update_test_checks.py \ --help -.PHONY: $(TESTS) $(addprefix update-,$(TESTS_ll)) check all . +clean: + rm -rf .lit_test_times.txt Output + +.PHONY: $(TESTS) $(addprefix update-,$(TESTS_ll)) check all clean update-help . diff --git a/test/llvmpasses/lower-handlers-addrspaces.ll b/test/llvmpasses/lower-handlers-addrspaces.ll index 8b85a71705f60..779caf9aa6bdf 100644 --- a/test/llvmpasses/lower-handlers-addrspaces.ll +++ b/test/llvmpasses/lower-handlers-addrspaces.ll @@ -8,8 +8,8 @@ target triple = "amdgcn-amd-amdhsa" target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7-ni:10:11:12:13" attributes #1 = { returns_twice } -declare i32 @julia.except_enter() #1 -declare void @ijl_pop_handler(i32) +declare i32 @julia.except_enter({}*) #1 +declare void @ijl_pop_handler({}*, i32) declare i8**** @julia.ptls_states() declare i8**** @julia.get_pgcstack() @@ -19,7 +19,7 @@ top: ; CHECK: call void @llvm.lifetime.start ; CHECK: call void @ijl_enter_handler ; CHECK: setjmp - %r = call i32 @julia.except_enter() + %r = call i32 @julia.except_enter({}* null) %cmp = icmp eq i32 %r, 0 br i1 %cmp, label %try, label %catch try: @@ -27,7 +27,7 @@ try: catch: br label %after after: - call void @ijl_pop_handler(i32 1) + call void @ijl_pop_handler({}* null, i32 1) ; CHECK: llvm.lifetime.end ret void } diff --git a/test/llvmpasses/lower-handlers.ll b/test/llvmpasses/lower-handlers.ll index a250edddcaa81..2361fb5b84ef6 100644 --- a/test/llvmpasses/lower-handlers.ll +++ b/test/llvmpasses/lower-handlers.ll @@ -5,8 +5,8 @@ ; RUN: opt -enable-new-pm=1 --opaque-pointers=1 --load-pass-plugin=libjulia-codegen%shlibext -passes='function(LowerExcHandlers)' -S %s | FileCheck %s attributes #1 = { returns_twice } -declare i32 @julia.except_enter() #1 -declare void @ijl_pop_handler(i32) +declare i32 @julia.except_enter({}*) #1 +declare void @ijl_pop_handler({}*, i32) declare i8**** @julia.ptls_states() declare i8**** @julia.get_pgcstack() @@ -16,7 +16,7 @@ top: ; CHECK: call void @llvm.lifetime.start ; CHECK: call void @ijl_enter_handler ; CHECK: setjmp - %r = call i32 @julia.except_enter() + %r = call i32 @julia.except_enter({}* null) %cmp = icmp eq i32 %r, 0 br i1 %cmp, label %try, label %catch try: @@ -24,7 +24,7 @@ try: catch: br label %after after: - call void @ijl_pop_handler(i32 1) + call void @ijl_pop_handler({}* null, i32 1) ; CHECK: llvm.lifetime.end ret void } From f54bb4c1e939ea5bf4aefabf95d3beedc9878521 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 14 Mar 2024 10:23:58 -0400 Subject: [PATCH 50/55] Increase build precompilation (#53682) - Bake precompilepkgs into the sysimage - Enable compilation in stdlib JLLs @staticfloat I guess this is ok now we have pkgimages? - Add some hardcoded precompiles encountered locally - Disable a problematic llvm test, with a note With this there's no compilation loading Pkg in interactive mode ``` % ./julia --trace-compile=stderr --start=no -q julia> ] (@v1.12) pkg> ``` Before ``` % time julia +nightly --start=no --trace-compile=stderr -q -e "using Pkg" precompile(Tuple{typeof(Base.setindex!), Base.Dict{Symbol, Union{Nothing, Bool, Int64, String}}, Int64, Symbol}) precompile(Tuple{typeof(Base.haskey), Base.Dict{String, Any}, String}) precompile(Tuple{typeof(MbedTLS_jll.__init__)}) precompile(Tuple{typeof(LibSSH2_jll.__init__)}) precompile(Tuple{typeof(LibGit2_jll.__init__)}) precompile(Tuple{typeof(nghttp2_jll.__init__)}) precompile(Tuple{typeof(LibCURL_jll.__init__)}) precompile(Tuple{typeof(MozillaCACerts_jll.__init__)}) precompile(Tuple{typeof(Base.Filesystem.normpath), String, String, Vararg{String}}) precompile(Tuple{typeof(p7zip_jll.__init__)}) precompile(Tuple{typeof(p7zip_jll.init_p7zip_path)}) precompile(Tuple{typeof(Base.append!), Array{String, 1}, Array{String, 1}}) precompile(Tuple{typeof(Base.join), Array{String, 1}, Char}) julia +nightly --start=no --trace-compile=stderr -q -e "using Pkg" 1.48s user 0.40s system 206% cpu 0.910 total ``` PR ``` % time ./julia --start=no --trace-compile=stderr -q -e "using Pkg" ./julia --start=no --trace-compile=stderr -q -e "using Pkg" 1.62s user 0.13s system 402% cpu 0.433 total ``` (cherry picked from commit c4ab0d46a9c3a470794e5a441176bfdf8ff36e52) --- base/precompilation.jl | 23 +++--- contrib/generate_precompile.jl | 77 ++++++++++++++----- .../src/CompilerSupportLibraries_jll.jl | 1 - stdlib/GMP_jll/src/GMP_jll.jl | 1 - stdlib/LLD_jll/src/LLD_jll.jl | 1 - .../src/LLVMLibUnwind_jll.jl | 1 - stdlib/LibCURL_jll/src/LibCURL_jll.jl | 1 - stdlib/LibGit2_jll/src/LibGit2_jll.jl | 1 - stdlib/LibSSH2_jll/src/LibSSH2_jll.jl | 1 - stdlib/LibUV_jll/src/LibUV_jll.jl | 1 - stdlib/LibUnwind_jll/src/LibUnwind_jll.jl | 1 - stdlib/MPFR_jll/src/MPFR_jll.jl | 1 - stdlib/MbedTLS_jll/src/MbedTLS_jll.jl | 1 - .../src/MozillaCACerts_jll.jl | 1 - stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl | 1 - stdlib/OpenLibm_jll/src/OpenLibm_jll.jl | 1 - stdlib/PCRE2_jll/src/PCRE2_jll.jl | 1 - stdlib/REPL/test/precompilation.jl | 5 +- stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl | 1 - stdlib/Zlib_jll/src/Zlib_jll.jl | 1 - stdlib/dSFMT_jll/src/dSFMT_jll.jl | 1 - stdlib/libLLVM_jll/src/libLLVM_jll.jl | 1 - .../src/libblastrampoline_jll.jl | 1 - stdlib/nghttp2_jll/src/nghttp2_jll.jl | 1 - stdlib/p7zip_jll/src/p7zip_jll.jl | 1 - test/cmdlineargs.jl | 8 +- 26 files changed, 77 insertions(+), 58 deletions(-) diff --git a/base/precompilation.jl b/base/precompilation.jl index 9ecb190a1afb5..f041ff044dd03 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -354,12 +354,10 @@ function precompilepkgs(pkgs::Vector{String}=String[]; _from_loading::Bool=false, configs::Union{Config,Vector{Config}}=(``=>Base.CacheFlags()), io::IO=stderr, - flags_cacheflags=nothing) + # asking for timing disables fancy mode, as timing is shown in non-fancy mode + fancyprint::Bool = can_fancyprint(io) && !timing + ) - if flags_cacheflags !== nothing - # TODO: Remove `flags_cacheflags` once Pkg moves to the renamed kwarg `configs` - configs = flags_cacheflags - end configs = configs isa Config ? [configs] : configs time_start = time_ns() @@ -374,9 +372,6 @@ function precompilepkgs(pkgs::Vector{String}=String[]; num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks))) parallel_limiter = Base.Semaphore(num_tasks) - # asking for timing disables fancy mode, as timing is shown in non-fancy mode - fancyprint = can_fancyprint(io) && !timing - if _from_loading && !Sys.isinteractive() && Base.get_bool_env("JULIA_TESTS", false) # suppress passive loading printing in julia test suite. `JULIA_TESTS` is set in Base.runtests io = devnull @@ -434,7 +429,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; end end - @debug "precompile: deps collected" _group=:precompile + @debug "precompile: deps collected" # this loop must be run after the full depsmap has been populated for (pkg, pkg_exts) in pkg_exts_map # find any packages that depend on the extension(s)'s deps and replace those deps in their deps list with the extension(s), @@ -446,7 +441,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; end end end - @debug "precompile: extensions collected" _group=:precompile + @debug "precompile: extensions collected" # return early if no deps if isempty(depsmap) @@ -474,7 +469,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; was_recompiled[dep_config] = false end end - @debug "precompile: signalling initialized" _group=:precompile + @debug "precompile: signalling initialized" # find and guard against circular deps @@ -515,7 +510,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; if !isempty(circular_deps) @warn """Circular dependency detected. Precompilation will be skipped for:\n $(join(string.(circular_deps), "\n "))""" end - @debug "precompile: circular dep check done" _group=:precompile + @debug "precompile: circular dep check done" # if a list of packages is given, restrict to dependencies of given packages if !isempty(pkgs) @@ -561,7 +556,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; else target *= "..." end - @debug "precompile: packages filtered" _group=:precompile + @debug "precompile: packages filtered" pkg_queue = PkgConfig[] failed_deps = Dict{PkgConfig, String}() @@ -736,7 +731,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; if !_from_loading Base.LOADING_CACHE[] = Base.LoadingCache() end - @debug "precompile: starting precompilation loop" _group=:precompile + @debug "precompile: starting precompilation loop" depsmap direct_deps ## precompilation loop for (pkg, deps) in depsmap cachepaths = Base.find_all_in_cache_path(pkg) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 9e4e2d4485189..92dde676244e4 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -39,6 +39,17 @@ precompile(Base.unsafe_string, (Ptr{Int8},)) # loading.jl precompile(Base.__require_prelocked, (Base.PkgId, Nothing)) precompile(Base._require, (Base.PkgId, Nothing)) +precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int)) +precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int)) + +# Pkg loading +precompile(Tuple{typeof(Base.Filesystem.normpath), String, String, Vararg{String}}) +precompile(Tuple{typeof(Base.append!), Array{String, 1}, Array{String, 1}}) +precompile(Tuple{typeof(Base.join), Array{String, 1}, Char}) +precompile(Tuple{typeof(Base.getindex), Base.Dict{Any, Any}, Char}) +precompile(Tuple{typeof(Base.delete!), Base.Set{Any}, Char}) +precompile(Tuple{typeof(Base.convert), Type{Base.Dict{String, Base.Dict{String, String}}}, Base.Dict{String, Any}}) +precompile(Tuple{typeof(Base.convert), Type{Base.Dict{String, Array{String, 1}}}, Base.Dict{String, Any}}) # REPL precompile(isequal, (String, String)) @@ -67,6 +78,10 @@ precompile(Tuple{typeof(haskey), Dict{Base.PkgId,Vector{Function}}, Base.PkgId}) precompile(Tuple{typeof(delete!), Dict{Base.PkgId,Vector{Function}}, Base.PkgId}) precompile(Tuple{typeof(push!), Vector{Function}, Function}) +# preferences +precompile(Base.get_preferences, (Base.UUID,)) +precompile(Base.record_compiletime_preference, (Base.UUID, String)) + # miscellaneous precompile(Tuple{typeof(Base.exit)}) precompile(Tuple{typeof(Base.require), Base.PkgId}) @@ -84,6 +99,23 @@ precompile(Base.CoreLogging.current_logger_for_env, (Base.CoreLogging.LogLevel, precompile(Base.CoreLogging.env_override_minlevel, (Symbol, Module)) precompile(Base.StackTraces.lookup, (Ptr{Nothing},)) precompile(Tuple{typeof(Base.run_module_init), Module, Int}) + +# precompilepkgs +precompile(Tuple{typeof(Base.get), Type{Array{String, 1}}, Base.Dict{String, Any}, String}) +precompile(Tuple{typeof(Base.get), Type{Base.Dict{String, Any}}, Base.Dict{String, Any}, String}) +precompile(Tuple{typeof(Base.haskey), Base.Dict{String, Any}, String}) +precompile(Tuple{typeof(Base.indexed_iterate), Tuple{Base.TTY, Bool}, Int, Int}) +precompile(Tuple{typeof(Base.indexed_iterate), Tuple{Base.TTY, Bool}, Int}) +precompile(Tuple{typeof(Base.open), Base.CmdRedirect, String, Base.TTY}) +precompile(Tuple{typeof(Base.Precompilation.precompilepkgs)}) +precompile(Tuple{typeof(Base.Precompilation.printpkgstyle), Base.TTY, Symbol, String}) +precompile(Tuple{typeof(Base.rawhandle), Base.TTY}) +precompile(Tuple{typeof(Base.setindex!), Base.Dict{String, Array{String, 1}}, Array{String, 1}, String}) +precompile(Tuple{typeof(Base.setindex!), GenericMemory{:not_atomic, Union{Base.Libc.RawFD, Base.SyncCloseFD, IO}, Core.AddrSpace{Core}(0x00)}, Base.TTY, Int}) +precompile(Tuple{typeof(Base.setup_stdio), Base.TTY, Bool}) +precompile(Tuple{typeof(Base.spawn_opts_inherit), Base.DevNull, Base.TTY, Base.TTY}) +precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:context,), Tuple{Base.TTY}}, typeof(Base.sprint), Function}) +precompile(Tuple{Type{Base.UUID}, Base.UUID}) """ for T in (Float16, Float32, Float64), IO in (IOBuffer, IOContext{IOBuffer}, Base.TTY, IOContext{Base.TTY}) @@ -131,17 +163,17 @@ for match = Base._methods(+, (Int, Int), -1, Base.get_world_counter()) push!(Expr[], Expr(:return, false)) vcat(String[], String[]) k, v = (:hello => nothing) - precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int)) - precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int)) + # Preferences uses these - precompile(Base.get_preferences, (Base.UUID,)) - precompile(Base.record_compiletime_preference, (Base.UUID, String)) get(Dict{String,Any}(), "missing", nothing) delete!(Dict{String,Any}(), "missing") for (k, v) in Dict{String,Any}() println(k) end + # interactive statup uses this + write(IOBuffer(), "") + break # only actually need to do this once end """ @@ -223,10 +255,10 @@ ansi_disablecursor = "\e[?25l" blackhole = Sys.isunix() ? "/dev/null" : "nul" procenv = Dict{String,Any}( "JULIA_HISTORY" => blackhole, - "JULIA_PROJECT" => nothing, # remove from environment - "JULIA_LOAD_PATH" => "@stdlib", + "JULIA_LOAD_PATH" => "@$(Sys.iswindows() ? ";" : ":")@stdlib", "JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":", "TERM" => "", + # "JULIA_DEBUG" => "precompilation", "JULIA_FALLBACK_REPL" => "true") generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printed @@ -263,24 +295,32 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe print_state("step1" => "R") # Also precompile a package here pkgname = "__PackagePrecompilationStatementModule" - mkpath(joinpath(prec_path, pkgname, "src")) - path = joinpath(prec_path, pkgname, "src", "$pkgname.jl") - write(path, - """ - module $pkgname - end - """) + pkguuid = "824efdaf-a0e9-431c-8ee7-3d356b2531c2" + pkgpath = joinpath(prec_path, pkgname) + mkpath(joinpath(pkgpath, "src")) + write(joinpath(pkgpath, "src", "$pkgname.jl"), + """ + module $pkgname + println("Precompiling $pkgname") + end + """) + write(joinpath(pkgpath, "Project.toml"), + """ + name = "$pkgname" + uuid = "$pkguuid" + """) + touch(joinpath(pkgpath, "Manifest.toml")) tmp_prec = tempname(prec_path) tmp_proc = tempname(prec_path) s = """ - pushfirst!(DEPOT_PATH, $(repr(prec_path))); + pushfirst!(DEPOT_PATH, $(repr(joinpath(prec_path,"depot")))); Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp_prec)); - Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path))) + Base.Precompilation.precompilepkgs(;fancyprint=true); $precompile_script """ p = run(pipeline(addenv(`$(julia_exepath()) -O0 --trace-compile=$tmp_proc --sysimage $sysimg - --cpu-target=native --startup-file=no --color=yes`, procenv), - stdin=IOBuffer(s), stdout=debug_output)) + --cpu-target=native --startup-file=no --color=yes --project=$(pkgpath)`, procenv), + stdin=IOBuffer(s), stderr=debug_output, stdout=debug_output)) n_step1 = 0 for f in (tmp_prec, tmp_proc) isfile(f) || continue @@ -334,9 +374,8 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe yield() # Make clock spinning print_state("step3" => string("R$n_succeeded", failed > 0 ? " ($failed failed)" : "")) catch ex - @show backtrace() # See #28808 - @warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0 + @warn "Failed to precompile expression" form=statement exception=(ex,catch_backtrace()) _module=nothing _file=nothing _line=0 end end wait(clock) # Stop asynchronous printing diff --git a/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl b/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl index bd7a0571f9d5a..a57b275b1862d 100644 --- a/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl +++ b/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl @@ -4,7 +4,6 @@ baremodule CompilerSupportLibraries_jll using Base, Libdl, Base.BinaryPlatforms -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/GMP_jll/src/GMP_jll.jl b/stdlib/GMP_jll/src/GMP_jll.jl index fde2fc15acf90..ae8b3c0b3e7d5 100644 --- a/stdlib/GMP_jll/src/GMP_jll.jl +++ b/stdlib/GMP_jll/src/GMP_jll.jl @@ -3,7 +3,6 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/GMP_jll.jl baremodule GMP_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/LLD_jll/src/LLD_jll.jl b/stdlib/LLD_jll/src/LLD_jll.jl index 55ccec9cc4005..9b8365dddcf0b 100644 --- a/stdlib/LLD_jll/src/LLD_jll.jl +++ b/stdlib/LLD_jll/src/LLD_jll.jl @@ -4,7 +4,6 @@ baremodule LLD_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl b/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl index 5c4026291a673..429e35b91d3f2 100644 --- a/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl +++ b/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl @@ -4,7 +4,6 @@ baremodule LLVMLibUnwind_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/LibCURL_jll/src/LibCURL_jll.jl b/stdlib/LibCURL_jll/src/LibCURL_jll.jl index cd67bfac0006a..3291c97d811cb 100644 --- a/stdlib/LibCURL_jll/src/LibCURL_jll.jl +++ b/stdlib/LibCURL_jll/src/LibCURL_jll.jl @@ -4,7 +4,6 @@ baremodule LibCURL_jll using Base, Libdl, nghttp2_jll -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/LibGit2_jll/src/LibGit2_jll.jl b/stdlib/LibGit2_jll/src/LibGit2_jll.jl index ff625a6494a26..c24d3c20f428e 100644 --- a/stdlib/LibGit2_jll/src/LibGit2_jll.jl +++ b/stdlib/LibGit2_jll/src/LibGit2_jll.jl @@ -4,7 +4,6 @@ baremodule LibGit2_jll using Base, Libdl, MbedTLS_jll, LibSSH2_jll -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl b/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl index a809f7a912d6b..351cbe0e3729b 100644 --- a/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl +++ b/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl @@ -4,7 +4,6 @@ baremodule LibSSH2_jll using Base, Libdl, MbedTLS_jll -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/LibUV_jll/src/LibUV_jll.jl b/stdlib/LibUV_jll/src/LibUV_jll.jl index 767f055eb019f..febc47f168ab9 100644 --- a/stdlib/LibUV_jll/src/LibUV_jll.jl +++ b/stdlib/LibUV_jll/src/LibUV_jll.jl @@ -4,7 +4,6 @@ baremodule LibUV_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false # NOTE: This file is currently empty, as we link libuv statically for now. diff --git a/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl b/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl index 12abeaf598151..f97b18443b6fd 100644 --- a/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl +++ b/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl @@ -4,7 +4,6 @@ baremodule LibUnwind_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/MPFR_jll/src/MPFR_jll.jl b/stdlib/MPFR_jll/src/MPFR_jll.jl index c184a9801102f..219ab0cad41be 100644 --- a/stdlib/MPFR_jll/src/MPFR_jll.jl +++ b/stdlib/MPFR_jll/src/MPFR_jll.jl @@ -3,7 +3,6 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/MPFR_jll.jl baremodule MPFR_jll using Base, Libdl, GMP_jll -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/MbedTLS_jll/src/MbedTLS_jll.jl b/stdlib/MbedTLS_jll/src/MbedTLS_jll.jl index e46da42a9a638..6367213e2c4ab 100644 --- a/stdlib/MbedTLS_jll/src/MbedTLS_jll.jl +++ b/stdlib/MbedTLS_jll/src/MbedTLS_jll.jl @@ -4,7 +4,6 @@ baremodule MbedTLS_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/MozillaCACerts_jll/src/MozillaCACerts_jll.jl b/stdlib/MozillaCACerts_jll/src/MozillaCACerts_jll.jl index 244c1204563d5..1d5df0236ae9e 100644 --- a/stdlib/MozillaCACerts_jll/src/MozillaCACerts_jll.jl +++ b/stdlib/MozillaCACerts_jll/src/MozillaCACerts_jll.jl @@ -4,7 +4,6 @@ baremodule MozillaCACerts_jll using Base -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl b/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl index a0c11ab047142..2f151f63f4413 100644 --- a/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl +++ b/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl @@ -13,7 +13,6 @@ using Base, Libdl, Base.BinaryPlatforms # using CompilerSupportLibraries_jll # Because of this however, we have to manually load the libraries we # _do_ care about, namely libgfortran -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl b/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl index f2dee45a279cd..297cd25512894 100644 --- a/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl +++ b/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl @@ -3,7 +3,6 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/OpenLibm_jll.jl baremodule OpenLibm_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/PCRE2_jll/src/PCRE2_jll.jl b/stdlib/PCRE2_jll/src/PCRE2_jll.jl index e7f685820830b..d825ac74db5a8 100644 --- a/stdlib/PCRE2_jll/src/PCRE2_jll.jl +++ b/stdlib/PCRE2_jll/src/PCRE2_jll.jl @@ -3,7 +3,6 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/PCRE2_jll.jl baremodule PCRE2_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/REPL/test/precompilation.jl b/stdlib/REPL/test/precompilation.jl index 2dcf78c114d9a..228cbd212a2c1 100644 --- a/stdlib/REPL/test/precompilation.jl +++ b/stdlib/REPL/test/precompilation.jl @@ -27,7 +27,10 @@ if !Sys.iswindows() tracecompile_out = read(f, String) close(ptm) # close after reading so we don't get precompiles from error shutdown - expected_precompiles = 1 + # given this test checks that startup is snappy, it's best to add workloads to + # contrib/generate_precompile.jl rather than increase this number. But if that's not + # possible, it'd be helpful to add a comment with the statement and a reason below + expected_precompiles = 0 n_precompiles = count(r"precompile\(", tracecompile_out) diff --git a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl index 08614880a934c..e5edfa76997e1 100644 --- a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl +++ b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl @@ -3,7 +3,6 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/SuiteSparse_jll.jl baremodule SuiteSparse_jll using Base, Libdl, libblastrampoline_jll -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/Zlib_jll/src/Zlib_jll.jl b/stdlib/Zlib_jll/src/Zlib_jll.jl index ea381b8b0683c..fb043c7143789 100644 --- a/stdlib/Zlib_jll/src/Zlib_jll.jl +++ b/stdlib/Zlib_jll/src/Zlib_jll.jl @@ -3,7 +3,6 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/Zlib_jll.jl baremodule Zlib_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/dSFMT_jll/src/dSFMT_jll.jl b/stdlib/dSFMT_jll/src/dSFMT_jll.jl index 35ada23778a94..b84bf0d8204ae 100644 --- a/stdlib/dSFMT_jll/src/dSFMT_jll.jl +++ b/stdlib/dSFMT_jll/src/dSFMT_jll.jl @@ -4,7 +4,6 @@ baremodule dSFMT_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/libLLVM_jll/src/libLLVM_jll.jl b/stdlib/libLLVM_jll/src/libLLVM_jll.jl index 3140dc3989a72..be2acb34faa65 100644 --- a/stdlib/libLLVM_jll/src/libLLVM_jll.jl +++ b/stdlib/libLLVM_jll/src/libLLVM_jll.jl @@ -4,7 +4,6 @@ baremodule libLLVM_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl b/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl index 49e7932a6b701..bbdad252be14a 100644 --- a/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl +++ b/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl @@ -4,7 +4,6 @@ baremodule libblastrampoline_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/nghttp2_jll/src/nghttp2_jll.jl b/stdlib/nghttp2_jll/src/nghttp2_jll.jl index 76e8d3582c402..5057299614aa5 100644 --- a/stdlib/nghttp2_jll/src/nghttp2_jll.jl +++ b/stdlib/nghttp2_jll/src/nghttp2_jll.jl @@ -3,7 +3,6 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/nghttp2_jll.jl baremodule nghttp2_jll using Base, Libdl -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/p7zip_jll/src/p7zip_jll.jl b/stdlib/p7zip_jll/src/p7zip_jll.jl index 01f26de936e78..a2a90a2450ea6 100644 --- a/stdlib/p7zip_jll/src/p7zip_jll.jl +++ b/stdlib/p7zip_jll/src/p7zip_jll.jl @@ -3,7 +3,6 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/p7zip_jll.jl baremodule p7zip_jll using Base -Base.Experimental.@compiler_options compile=min optimize=0 infer=false const PATH_list = String[] const LIBPATH_list = String[] diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 498254bb26ce0..7cc51520fb0f2 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -677,7 +677,9 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` code = code[3] @test occursin("llvm.module.flags", code) @test occursin("llvm.dbg.cu", code) - @test occursin("int.jl", code) + # TODO: consider moving test to llvmpasses as this fails on some platforms + # without clear reason + @test_skip occursin("int.jl", code) @test !occursin("name: \"Int64\"", code) end let code = readchomperrors(`$exename -g2 -E "@eval Int64(1)+Int64(1)"`) @@ -685,7 +687,9 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` code = code[3] @test occursin("llvm.module.flags", code) @test occursin("llvm.dbg.cu", code) - @test occursin("int.jl", code) + # TODO: consider moving test to llvmpasses as this fails on some platforms + # without clear reason + @test_skip occursin("int.jl", code) @test occursin("name: \"Int64\"", code) end end From b6c957b463cb39c1c16400fc07b177fbf896b763 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Mon, 11 Mar 2024 11:44:28 +0530 Subject: [PATCH 51/55] Fix linear indexing for ReshapedArray if the parent has offset axes (#41232) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR fixes ```julia julia> r = reshape(Base.IdentityUnitRange(3:4), 2, 1) 2×1 reshape(::Base.IdentityUnitRange{UnitRange{Int64}}, 2, 1) with eltype Int64: 3 4 julia> collect(r) == r false julia> collect(r) 2×1 Matrix{Int64}: 3258125826116431922 3688512103538242609 ``` After this PR, ```julia julia> collect(r) 2×1 Matrix{Int64}: 3 4 ``` (cherry picked from commit 6e3044d46acc90b7ba948bef7bd0c276f5817870) --- base/reshapedarray.jl | 9 ++++++--- test/abstractarray.jl | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index 344858e76764a..63e508e8835de 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -238,7 +238,8 @@ offset_if_vec(i::Integer, axs::Tuple) = i @inline function isassigned(A::ReshapedArrayLF, index::Int) @boundscheck checkbounds(Bool, A, index) || return false - @inbounds ret = isassigned(parent(A), index) + indexparent = index - firstindex(A) + firstindex(parent(A)) + @inbounds ret = isassigned(parent(A), indexparent) ret end @inline function isassigned(A::ReshapedArray{T,N}, indices::Vararg{Int, N}) where {T,N} @@ -251,7 +252,8 @@ end @inline function getindex(A::ReshapedArrayLF, index::Int) @boundscheck checkbounds(A, index) - @inbounds ret = parent(A)[index] + indexparent = index - firstindex(A) + firstindex(parent(A)) + @inbounds ret = parent(A)[indexparent] ret end @inline function getindex(A::ReshapedArray{T,N}, indices::Vararg{Int,N}) where {T,N} @@ -275,7 +277,8 @@ end @inline function setindex!(A::ReshapedArrayLF, val, index::Int) @boundscheck checkbounds(A, index) - @inbounds parent(A)[index] = val + indexparent = index - firstindex(A) + firstindex(parent(A)) + @inbounds parent(A)[indexparent] = val val end @inline function setindex!(A::ReshapedArray{T,N}, val, indices::Vararg{Int,N}) where {T,N} diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 15f8ed638bcc5..732ffba85ae84 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -2029,3 +2029,33 @@ end @test B == A end end + +@testset "reshape for offset arrays" begin + p = Base.IdentityUnitRange(3:4) + r = reshape(p, :, 1) + @test r[eachindex(r)] == UnitRange(p) + @test collect(r) == r + + struct ZeroBasedArray{T,N,A<:AbstractArray{T,N}} <: AbstractArray{T,N} + a :: A + function ZeroBasedArray(a::AbstractArray) + Base.require_one_based_indexing(a) + new{eltype(a), ndims(a), typeof(a)}(a) + end + end + Base.parent(z::ZeroBasedArray) = z.a + Base.size(z::ZeroBasedArray) = size(parent(z)) + Base.axes(z::ZeroBasedArray) = map(x -> Base.IdentityUnitRange(0:x - 1), size(parent(z))) + Base.getindex(z::ZeroBasedArray{<:Any, N}, i::Vararg{Int,N}) where {N} = parent(z)[map(x -> x + 1, i)...] + Base.setindex!(z::ZeroBasedArray{<:Any, N}, val, i::Vararg{Int,N}) where {N} = parent(z)[map(x -> x + 1, i)...] = val + + z = ZeroBasedArray(collect(1:4)) + r2 = reshape(z, :, 1) + @test r2[CartesianIndices(r2)] == r2[LinearIndices(r2)] + r2[firstindex(r2)] = 34 + @test z[0] == 34 + r2[eachindex(r2)] = r2 .* 2 + for (i, j) in zip(eachindex(r2), eachindex(z)) + @test r2[i] == z[j] + end +end From 22db541f26edfb43a5768f412280ac347f3801d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Mon, 26 Feb 2024 01:49:17 +0000 Subject: [PATCH 52/55] Use `@test ... skip=...` instead of `@test_skip` (cherry picked from commit 34a515189a653080776e9d24295a0cfee393fe90) --- test/file.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/file.jl b/test/file.jl index a476895af442a..7b116ea4c13e8 100644 --- a/test/file.jl +++ b/test/file.jl @@ -1685,19 +1685,19 @@ end chmod(fpath, 0o644) @test !Sys.isexecutable(fpath) @test Sys.isreadable(fpath) - Sys.iswindows() ? @test_skip(Sys.iswriteable(fpath)) : @test(Sys.iswriteable(fpath)) + @test Sys.iswriteable(fpath) skip=Sys.iswindows() chmod(fpath, 0o755) @test Sys.isexecutable(fpath) @test Sys.isreadable(fpath) - Sys.iswindows() ? @test_skip(Sys.iswriteable(fpath)) : @test(Sys.iswriteable(fpath)) + @test Sys.iswriteable(fpath) skip=Sys.iswindows() chmod(fpath, 0o444) @test !Sys.isexecutable(fpath) @test Sys.isreadable(fpath) @test !Sys.iswriteable(fpath) chmod(fpath, 0o244) @test !Sys.isexecutable(fpath) - Sys.iswindows() ? @test_skip(!Sys.isreadable(fpath)) : @test(!Sys.isreadable(fpath)) - Sys.iswindows() ? @test_skip(Sys.iswriteable(fpath)) : @test(Sys.iswriteable(fpath)) + @test !Sys.isreadable(fpath) skip=Sys.iswindows() + @test Sys.iswriteable(fpath) skip=Sys.iswindows() # Ensure that, on Windows, where inheritance is default, # chmod still behaves as we expect. From b9aaff5c9583eff30efe17db27671c11cd9f0f75 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 13 Mar 2024 10:04:08 +0100 Subject: [PATCH 53/55] Move `isexecutable, isreadable, iswritable` to `filesystem.jl` (#53699) This PR migrates the methods `isexecutable, isreadable, iswritable` from `Sys` to `Base`, but also generates an alias in `Sys` for backwards compatibility. Furthermore, `iswriteable` is renamed to `iswritable` in order to match the already existing `Base.iswritable` method. Suggested in https://github.com/JuliaLang/julia/pull/53320#issuecomment-1989217973. (cherry picked from commit 8d31f334269c6553553cad4220215a03c6da7ab3) --- base/exports.jl | 1 + base/filesystem.jl | 83 ++++++++++++++++++++++++++++++++++++- base/io.jl | 2 + base/sysinfo.jl | 84 ++------------------------------------ doc/src/base/base.md | 2 +- doc/src/base/io-network.md | 1 + test/file.jl | 12 +++--- 7 files changed, 97 insertions(+), 88 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index ba022082ced35..88adfb9bfba76 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -930,6 +930,7 @@ export isblockdev, ischardev, isdir, + isexecutable, isfifo, isfile, islink, diff --git a/base/filesystem.jl b/base/filesystem.jl index ee6fde982caab..36e4bb4596285 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -140,7 +140,8 @@ import .Base: IOError, _UVError, _sizeof_uv_fs, check_open, close, eof, eventloop, fd, isopen, bytesavailable, position, read, read!, readavailable, seek, seekend, show, skip, stat, unsafe_read, unsafe_write, write, transcode, uv_error, - setup_stdio, rawhandle, OS_HANDLE, INVALID_OS_HANDLE, windowserror, filesize + setup_stdio, rawhandle, OS_HANDLE, INVALID_OS_HANDLE, windowserror, filesize, + isexecutable, isreadable, iswritable import .Base.RefValue @@ -365,5 +366,85 @@ function touch(f::File) f end +""" + isexecutable(path::String) + +Return `true` if the given `path` has executable permissions. + +!!! note + This permission may change before the user executes `path`, + so it is recommended to execute the file and handle the error if that fails, + rather than calling `isexecutable` first. + +!!! note + Prior to Julia 1.6, this did not correctly interrogate filesystem + ACLs on Windows, therefore it would return `true` for any + file. From Julia 1.6 on, it correctly determines whether the + file is marked as executable or not. + +See also [`ispath`](@ref), [`isreadable`](@ref), [`iswritable`](@ref). +""" +function isexecutable(path::String) + # We use `access()` and `X_OK` to determine if a given path is + # executable by the current user. `X_OK` comes from `unistd.h`. + X_OK = 0x01 + return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, X_OK) == 0 +end +isexecutable(path::AbstractString) = isexecutable(String(path)) + +""" + isreadable(path::String) + +Return `true` if the access permissions for the given `path` permitted reading by the current user. + +!!! note + This permission may change before the user calls `open`, + so it is recommended to just call `open` alone and handle the error if that fails, + rather than calling `isreadable` first. + +!!! note + Currently this function does not correctly interrogate filesystem + ACLs on Windows, therefore it can return wrong results. + +!!! compat "Julia 1.11" + This function requires at least Julia 1.11. + +See also [`ispath`](@ref), [`isexecutable`](@ref), [`iswritable`](@ref). +""" +function isreadable(path::String) + # We use `access()` and `R_OK` to determine if a given path is + # readable by the current user. `R_OK` comes from `unistd.h`. + R_OK = 0x04 + return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, R_OK) == 0 +end +isreadable(path::AbstractString) = isreadable(String(path)) + +""" + iswritable(path::String) + +Return `true` if the access permissions for the given `path` permitted writing by the current user. + +!!! note + This permission may change before the user calls `open`, + so it is recommended to just call `open` alone and handle the error if that fails, + rather than calling `iswritable` first. + +!!! note + Currently this function does not correctly interrogate filesystem + ACLs on Windows, therefore it can return wrong results. + +!!! compat "Julia 1.11" + This function requires at least Julia 1.11. + +See also [`ispath`](@ref), [`isexecutable`](@ref), [`isreadable`](@ref). +""" +function iswritable(path::String) + # We use `access()` and `W_OK` to determine if a given path is + # writeable by the current user. `W_OK` comes from `unistd.h`. + W_OK = 0x02 + return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, W_OK) == 0 +end +iswritable(path::AbstractString) = iswritable(String(path)) + end diff --git a/base/io.jl b/base/io.jl index 7d93bc3e6d9c8..fb883234be4df 100644 --- a/base/io.jl +++ b/base/io.jl @@ -131,6 +131,8 @@ data has already been buffered. The result is a `Vector{UInt8}`. """ function readavailable end +function isexecutable end + """ isreadable(io) -> Bool diff --git a/base/sysinfo.jl b/base/sysinfo.jl index c5744873312d6..9bdff3673477f 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -34,7 +34,7 @@ export BINDIR, isjsvm, isexecutable, isreadable, - iswriteable, + iswritable, username, which @@ -553,85 +553,9 @@ windows_version const WINDOWS_VISTA_VER = v"6.0" -""" - Sys.isexecutable(path::String) - -Return `true` if the given `path` has executable permissions. - -!!! note - This permission may change before the user executes `path`, - so it is recommended to execute the file and handle the error if that fails, - rather than calling `isexecutable` first. - -!!! note - Prior to Julia 1.6, this did not correctly interrogate filesystem - ACLs on Windows, therefore it would return `true` for any - file. From Julia 1.6 on, it correctly determines whether the - file is marked as executable or not. - -See also [`ispath`](@ref), [`isreadable`](@ref), [`iswriteable`](@ref). -""" -function isexecutable(path::String) - # We use `access()` and `X_OK` to determine if a given path is - # executable by the current user. `X_OK` comes from `unistd.h`. - X_OK = 0x01 - return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, X_OK) == 0 -end -isexecutable(path::AbstractString) = isexecutable(String(path)) - -""" - Sys.isreadable(path::String) - -Return `true` if the access permissions for the given `path` permitted reading by the current user. - -!!! note - This permission may change before the user calls `open`, - so it is recommended to just call `open` alone and handle the error if that fails, - rather than calling `isreadable` first. - -!!! note - Currently this function does not correctly interrogate filesystem - ACLs on Windows, therefore it can return wrong results. - -!!! compat "Julia 1.11" - This function requires at least Julia 1.11. - -See also [`ispath`](@ref), [`isexecutable`](@ref), [`iswriteable`](@ref). -""" -function isreadable(path::String) - # We use `access()` and `R_OK` to determine if a given path is - # readable by the current user. `R_OK` comes from `unistd.h`. - R_OK = 0x04 - return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, R_OK) == 0 -end -isreadable(path::AbstractString) = isreadable(String(path)) - -""" - Sys.iswriteable(path::String) - -Return `true` if the access permissions for the given `path` permitted writing by the current user. - -!!! note - This permission may change before the user calls `open`, - so it is recommended to just call `open` alone and handle the error if that fails, - rather than calling `iswriteable` first. - -!!! note - Currently this function does not correctly interrogate filesystem - ACLs on Windows, therefore it can return wrong results. - -!!! compat "Julia 1.11" - This function requires at least Julia 1.11. - -See also [`ispath`](@ref), [`isexecutable`](@ref), [`isreadable`](@ref). -""" -function iswriteable(path::String) - # We use `access()` and `W_OK` to determine if a given path is - # writeable by the current user. `W_OK` comes from `unistd.h`. - W_OK = 0x02 - return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, W_OK) == 0 -end -iswriteable(path::AbstractString) = iswriteable(String(path)) +const isexecutable = Base.isexecutable +const isreadable = Base.isreadable +const iswritable = Base.iswritable """ Sys.which(program_name::String) diff --git a/doc/src/base/base.md b/doc/src/base/base.md index 556d51d3af0c1..739ee97d1dd43 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -375,7 +375,7 @@ Base.Sys.isjsvm Base.Sys.loadavg Base.Sys.isexecutable Base.Sys.isreadable -Base.Sys.iswriteable +Base.Sys.iswritable Base.Sys.username Base.@static ``` diff --git a/doc/src/base/io-network.md b/doc/src/base/io-network.md index f99f592e0fb5c..123d494c396a0 100644 --- a/doc/src/base/io-network.md +++ b/doc/src/base/io-network.md @@ -39,6 +39,7 @@ Base.eof Base.isreadonly Base.iswritable Base.isreadable +Base.isexecutable Base.isopen Base.fd Base.redirect_stdio diff --git a/test/file.jl b/test/file.jl index 7b116ea4c13e8..925b1e0ad5905 100644 --- a/test/file.jl +++ b/test/file.jl @@ -1668,7 +1668,7 @@ else ) end -@testset "chmod/isexecutable/isreadable/iswriteable" begin +@testset "chmod/isexecutable/isreadable/iswritable" begin mktempdir() do dir subdir = joinpath(dir, "subdir") fpath = joinpath(dir, "subdir", "foo") @@ -1685,19 +1685,19 @@ end chmod(fpath, 0o644) @test !Sys.isexecutable(fpath) @test Sys.isreadable(fpath) - @test Sys.iswriteable(fpath) skip=Sys.iswindows() + @test Sys.iswritable(fpath) skip=Sys.iswindows() chmod(fpath, 0o755) @test Sys.isexecutable(fpath) @test Sys.isreadable(fpath) - @test Sys.iswriteable(fpath) skip=Sys.iswindows() + @test Sys.iswritable(fpath) skip=Sys.iswindows() chmod(fpath, 0o444) @test !Sys.isexecutable(fpath) @test Sys.isreadable(fpath) - @test !Sys.iswriteable(fpath) + @test !Sys.iswritable(fpath) chmod(fpath, 0o244) @test !Sys.isexecutable(fpath) @test !Sys.isreadable(fpath) skip=Sys.iswindows() - @test Sys.iswriteable(fpath) skip=Sys.iswindows() + @test Sys.iswritable(fpath) skip=Sys.iswindows() # Ensure that, on Windows, where inheritance is default, # chmod still behaves as we expect. @@ -1705,7 +1705,7 @@ end chmod(subdir, 0o666) @test !Sys.isexecutable(fpath) @test Sys.isreadable(fpath) - @test_skip Sys.iswriteable(fpath) + @test_skip Sys.iswritable(fpath) end # Reset permissions to all at the end, so it can be deleted properly. From b5e41046410560e08e0c323a151331501dae9d50 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 16 Mar 2024 21:24:31 -0400 Subject: [PATCH 54/55] update Pkg release branch --- stdlib/Pkg.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 64a456f678ad1..93bd1ea305f46 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ -PKG_BRANCH = master +PKG_BRANCH = release-1.11 PKG_SHA1 = 1e6fa605f5554f37d2ebf47de9763152f6f78232 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From d6d84edd3020e732baf5b85969ded0c9f28583f4 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sat, 16 Mar 2024 22:04:56 -0400 Subject: [PATCH 55/55] =?UTF-8?q?=F0=9F=A4=96=20[backports-release-1.11]?= =?UTF-8?q?=20Bump=20the=20Pkg=20stdlib=20from=201e6fa605f=20to=20bd787952?= =?UTF-8?q?e=20(#53762)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: Pkg URL: https://github.com/JuliaLang/Pkg.jl.git Stdlib branch: release-1.11 Julia branch: backports-release-1.11 Old commit: 1e6fa605f New commit: bd787952e Julia version: 1.11.0-alpha1 Pkg version: 1.11.0 Bump invoked by: @IanButterworth Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/Pkg.jl/compare/1e6fa605f5554f37d2ebf47de9763152f6f78232...bd787952e6ceab2d3e8fec98429d954860ecfd9f ``` $ git log --oneline 1e6fa605f..bd787952e bd787952e Merge pull request #3846 from JuliaLang/backports-release-1.11 239702022 precompile: update kwargs (#3838) 4ea3f3576 move threads assignment after precompilation (#3840) ``` Co-authored-by: Dilum Aluthge --- .../Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/md5 | 1 - .../Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/sha512 | 1 - .../Pkg-bd787952e6ceab2d3e8fec98429d954860ecfd9f.tar.gz/md5 | 1 + .../Pkg-bd787952e6ceab2d3e8fec98429d954860ecfd9f.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-bd787952e6ceab2d3e8fec98429d954860ecfd9f.tar.gz/md5 create mode 100644 deps/checksums/Pkg-bd787952e6ceab2d3e8fec98429d954860ecfd9f.tar.gz/sha512 diff --git a/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/md5 b/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/md5 deleted file mode 100644 index 32abf7f99b463..0000000000000 --- a/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -f9e5cafbfda75a03d700b165c03e5bb3 diff --git a/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/sha512 b/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/sha512 deleted file mode 100644 index 5fa17a3e0fdb5..0000000000000 --- a/deps/checksums/Pkg-1e6fa605f5554f37d2ebf47de9763152f6f78232.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -3353a9da04aae86b788209afd19ace7132d94fca4f777f6f13841630a1eb4c63fd9020e53a38b269c3d4f1d1548dafa848edc5b9081a34852afe1b3a0eb874ff diff --git a/deps/checksums/Pkg-bd787952e6ceab2d3e8fec98429d954860ecfd9f.tar.gz/md5 b/deps/checksums/Pkg-bd787952e6ceab2d3e8fec98429d954860ecfd9f.tar.gz/md5 new file mode 100644 index 0000000000000..09eba0d1a7f51 --- /dev/null +++ b/deps/checksums/Pkg-bd787952e6ceab2d3e8fec98429d954860ecfd9f.tar.gz/md5 @@ -0,0 +1 @@ +3b2aa3606351309b2e5ca8b489c88c94 diff --git a/deps/checksums/Pkg-bd787952e6ceab2d3e8fec98429d954860ecfd9f.tar.gz/sha512 b/deps/checksums/Pkg-bd787952e6ceab2d3e8fec98429d954860ecfd9f.tar.gz/sha512 new file mode 100644 index 0000000000000..8f802b6c54f1a --- /dev/null +++ b/deps/checksums/Pkg-bd787952e6ceab2d3e8fec98429d954860ecfd9f.tar.gz/sha512 @@ -0,0 +1 @@ +9a58efdaf279b857fff3d446325e66547b0971adcdf3499764561235a32d30af72db8f3e11ebb4161ad1d502c4e59b6d6f0b0da48f941a04a0c333cda505b7ba diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 93bd1ea305f46..986b0b16e0b76 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = release-1.11 -PKG_SHA1 = 1e6fa605f5554f37d2ebf47de9763152f6f78232 +PKG_SHA1 = bd787952e6ceab2d3e8fec98429d954860ecfd9f PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1