From 050ecea82945af3f05b31393f5a6d4d6b796e9fa Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 24 Sep 2024 16:42:36 -0400 Subject: [PATCH 01/15] Mmap: fix grow! for non file IOs (#55849) Fixes https://github.com/JuliaLang/julia/issues/54203 Requires #55641 Based on https://github.com/JuliaLang/julia/pull/55641#issuecomment-2334162489 cc. @JakeZw @ronisbr --------- Co-authored-by: Jameson Nash (cherry picked from commit b0db75d7be83aa1a019e0801b30bb647d1d3e01e) --- stdlib/Mmap/src/Mmap.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index e6987582bf511..df2f4f1a19991 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -86,6 +86,8 @@ grow!(::Anonymous,o::Integer,l::Integer) = return function grow!(io::IO, offset::Integer, len::Integer) pos = position(io) filelen = filesize(io) + # If non-regular file skip trying to grow since we know that will fail the ftruncate syscall + filelen == 0 && !isfile(io) && return if filelen < offset + len failure = ccall(:jl_ftruncate, Cint, (Cint, Int64), fd(io), offset+len) Base.systemerror(:ftruncate, failure != 0) @@ -218,7 +220,7 @@ function mmap(io::IO, # platform-specific mmapping @static if Sys.isunix() prot, flags, iswrite = settings(file_desc, shared) - if requestedSizeLarger + if requestedSizeLarger && isfile(io) # add a condition to this line to ensure it only checks files if iswrite if grow grow!(io, offset, len) From 5f46099ae4147d67da31326e4173b557dc89a009 Mon Sep 17 00:00:00 2001 From: Daniel Wennberg Date: Wed, 25 Sep 2024 05:26:48 -0700 Subject: [PATCH 02/15] Update TaskLocalRNG docstring according to #49110 (#55863) Since #49110, which is included in 1.10 and 1.11, spawning a task no longer advances the parent task's RNG state, so this statement in the docs was incorrect. (cherry picked from commit c601b113be8f81a00711455349a3dfab7755de9d) --- stdlib/Random/src/Xoshiro.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/stdlib/Random/src/Xoshiro.jl b/stdlib/Random/src/Xoshiro.jl index 5569d6d5c1da5..adb8b158d78d4 100644 --- a/stdlib/Random/src/Xoshiro.jl +++ b/stdlib/Random/src/Xoshiro.jl @@ -185,8 +185,8 @@ end TaskLocalRNG The `TaskLocalRNG` has state that is local to its task, not its thread. -It is seeded upon task creation, from the state of its parent task. -Therefore, task creation is an event that changes the parent's RNG state. +It is seeded upon task creation, from the state of its parent task, but without +advancing the state of the parent's RNG. As an upside, the `TaskLocalRNG` is pretty fast, and permits reproducible multithreaded simulations (barring race conditions), independent of scheduler @@ -203,6 +203,9 @@ may be any integer. !!! compat "Julia 1.11" Seeding `TaskLocalRNG()` with a negative integer seed requires at least Julia 1.11. + +!!! compat "Julia 1.10" + Task creation no longer advances the parent task's RNG state as of Julia 1.10. """ struct TaskLocalRNG <: AbstractRNG end TaskLocalRNG(::Nothing) = TaskLocalRNG() From 7c2f5336b5d1373774fd5d362636aab62a643dd4 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 25 Sep 2024 09:40:06 -0300 Subject: [PATCH 03/15] Root globals in toplevel exprs (#54433) This fixes #54422, the code here assumes that top level exprs are always rooted, but I don't see that referenced anywhere else, or guaranteed, so conservatively always root objects that show up in code. (cherry picked from commit 6e5e87b2cafda840b90347c2e74202d2608d7c29) --- src/codegen.cpp | 5 +++-- test/gc.jl | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index aca3f8e2c5cd1..951cc1c86f9ee 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6144,8 +6144,9 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ jl_value_t *val = expr; if (jl_is_quotenode(expr)) val = jl_fieldref_noalloc(expr, 0); - if (jl_is_method(ctx.linfo->def.method)) // toplevel exprs are already rooted - val = jl_ensure_rooted(ctx, val); + // Toplevel exprs are rooted but because codegen assumes this is constant, it removes the write barriers for this code. + // This means we have to globally root the value here. (The other option would be to change how we optimize toplevel code) + val = jl_ensure_rooted(ctx, val); return mark_julia_const(ctx, val); } diff --git a/test/gc.jl b/test/gc.jl index 578fe799fb935..866a95cb62626 100644 --- a/test/gc.jl +++ b/test/gc.jl @@ -39,3 +39,17 @@ end @testset "Base.GC docstrings" begin @test isempty(Docs.undocumented_names(GC)) end + +#testset doesn't work here because this needs to run in top level +#Check that we ensure objects in toplevel exprs are rooted +global dims54422 = [] # allocate the Binding +GC.gc(); GC.gc(); # force the binding to be old +GC.enable(false); # prevent new objects from being old +@eval begin + Base.Experimental.@force_compile # use the compiler + dims54422 = $([]) + nothing +end +GC.enable(true); GC.gc(false) # incremental collection +@test typeof(dims54422) == Vector{Any} +@test isempty(dims54422) From 969d816cd7aa81388182d6e94631a0b9c9e49b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:34:03 +0100 Subject: [PATCH 04/15] Fix some corner cases of `isapprox` with unsigned integers (#55828) (cherry picked from commit e4b29f71e7ca0e033ff3510b06d7534e4045e068) --- base/floatfuncs.jl | 4 +++- test/floatfuncs.jl | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/base/floatfuncs.jl b/base/floatfuncs.jl index a2a0f60bcf399..e6f7890d3adc7 100644 --- a/base/floatfuncs.jl +++ b/base/floatfuncs.jl @@ -232,7 +232,9 @@ function isapprox(x::Integer, y::Integer; if norm === abs && atol < 1 && rtol == 0 return x == y else - return norm(x - y) <= max(atol, rtol*max(norm(x), norm(y))) + # We need to take the difference `max` - `min` when comparing unsigned integers. + _x, _y = x < y ? (x, y) : (y, x) + return norm(_y - _x) <= max(atol, rtol*max(norm(_x), norm(_y))) end end diff --git a/test/floatfuncs.jl b/test/floatfuncs.jl index f33ec75b58322..d5d697634bcfa 100644 --- a/test/floatfuncs.jl +++ b/test/floatfuncs.jl @@ -257,6 +257,35 @@ end end end +@testset "isapprox and unsigned integers" begin + for T in Base.BitUnsigned_types + # Test also combinations of different integer types + W = widen(T) + # The order of the operands for difference between unsigned integers is + # very important, test both combinations. + @test isapprox(T(42), T(42); rtol=T(0), atol=0.5) + @test isapprox(T(42), W(42); rtol=T(0), atol=0.5) + @test !isapprox(T(0), T(1); rtol=T(0), atol=0.5) + @test !isapprox(T(1), T(0); rtol=T(0), atol=0.5) + @test isapprox(T(1), T(3); atol=T(2)) + @test isapprox(T(4), T(2); atol=T(2)) + @test isapprox(T(1), W(3); atol=T(2)) + @test isapprox(T(4), W(2); atol=T(2)) + @test isapprox(T(5), T(7); atol=typemax(T)) + @test isapprox(T(8), T(6); atol=typemax(T)) + @test isapprox(T(1), T(2); rtol=1) + @test isapprox(T(6), T(3); rtol=1) + @test isapprox(T(1), W(2); rtol=1) + @test isapprox(T(6), W(3); rtol=1) + @test !isapprox(typemin(T), typemax(T)) + @test !isapprox(typemax(T), typemin(T)) + @test !isapprox(typemin(T), typemax(T); atol=typemax(T)-T(1)) + @test !isapprox(typemax(T), typemin(T); atol=typemax(T)-T(1)) + @test isapprox(typemin(T), typemax(T); atol=typemax(T)) + @test isapprox(typemax(T), typemin(T); atol=typemax(T)) + end +end + @testset "Conversion from floating point to unsigned integer near extremes (#51063)" begin @test_throws InexactError UInt32(4.2949673f9) @test_throws InexactError UInt64(1.8446744f19) From eb8845f518cf7865f8427a701dcd7a841fa4f06b Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 26 Sep 2024 21:06:46 -0400 Subject: [PATCH 05/15] Profile: fix order of fields in heapsnapshot & improve formatting (#55890) (cherry picked from commit 32b9e1ac9fa31019aa3779b3c401a80bc94cb61f) --- src/gc-heap-snapshot.cpp | 42 ++++++++----------- stdlib/Profile/src/heapsnapshot_reassemble.jl | 10 ++++- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/src/gc-heap-snapshot.cpp b/src/gc-heap-snapshot.cpp index 77a6e70a127e6..cebcdf5b3c3f9 100644 --- a/src/gc-heap-snapshot.cpp +++ b/src/gc-heap-snapshot.cpp @@ -607,38 +607,32 @@ void final_serialize_heap_snapshot(ios_t *json, ios_t *strings, HeapSnapshot &sn { // mimicking https://github.com/nodejs/node/blob/5fd7a72e1c4fbaf37d3723c4c81dce35c149dc84/deps/v8/src/profiler/heap-snapshot-generator.cc#L2567-L2567 // also https://github.com/microsoft/vscode-v8-heap-tools/blob/c5b34396392397925ecbb4ecb904a27a2754f2c1/v8-heap-parser/src/decoder.rs#L43-L51 - ios_printf(json, "{\"snapshot\":{"); + ios_printf(json, "{\"snapshot\":{\n"); - ios_printf(json, "\"meta\":{"); - ios_printf(json, "\"node_fields\":[\"type\",\"name\",\"id\",\"self_size\",\"edge_count\",\"trace_node_id\",\"detachedness\"],"); - ios_printf(json, "\"node_types\":["); + ios_printf(json, " \"meta\":{\n"); + ios_printf(json, " \"node_fields\":[\"type\",\"name\",\"id\",\"self_size\",\"edge_count\",\"trace_node_id\",\"detachedness\"],\n"); + ios_printf(json, " \"node_types\":["); snapshot.node_types.print_json_array(json, false); ios_printf(json, ","); - ios_printf(json, "\"string\", \"number\", \"number\", \"number\", \"number\", \"number\"],"); - ios_printf(json, "\"edge_fields\":[\"type\",\"name_or_index\",\"to_node\"],"); - ios_printf(json, "\"edge_types\":["); + ios_printf(json, "\"string\", \"number\", \"number\", \"number\", \"number\", \"number\"],\n"); + ios_printf(json, " \"edge_fields\":[\"type\",\"name_or_index\",\"to_node\"],\n"); + ios_printf(json, " \"edge_types\":["); snapshot.edge_types.print_json_array(json, false); ios_printf(json, ","); - ios_printf(json, "\"string_or_number\",\"from_node\"],"); + ios_printf(json, "\"string_or_number\",\"from_node\"],\n"); // not used. Required by microsoft/vscode-v8-heap-tools - ios_printf(json, "\"trace_function_info_fields\":[\"function_id\",\"name\",\"script_name\",\"script_id\",\"line\",\"column\"],"); - ios_printf(json, "\"trace_node_fields\":[\"id\",\"function_info_index\",\"count\",\"size\",\"children\"],"); - ios_printf(json, "\"sample_fields\":[\"timestamp_us\",\"last_assigned_id\"],"); - ios_printf(json, "\"location_fields\":[\"object_index\",\"script_id\",\"line\",\"column\"]"); + ios_printf(json, " \"trace_function_info_fields\":[\"function_id\",\"name\",\"script_name\",\"script_id\",\"line\",\"column\"],\n"); + ios_printf(json, " \"trace_node_fields\":[\"id\",\"function_info_index\",\"count\",\"size\",\"children\"],\n"); + ios_printf(json, " \"sample_fields\":[\"timestamp_us\",\"last_assigned_id\"],\n"); + ios_printf(json, " \"location_fields\":[\"object_index\",\"script_id\",\"line\",\"column\"]\n"); // end not used - ios_printf(json, "},\n"); // end "meta" + ios_printf(json, " },\n"); // end "meta" - ios_printf(json, "\"node_count\":%zu,", snapshot.num_nodes); - ios_printf(json, "\"edge_count\":%zu,", snapshot.num_edges); - ios_printf(json, "\"trace_function_count\":0"); // not used. Required by microsoft/vscode-v8-heap-tools - ios_printf(json, "},\n"); // end "snapshot" - - // not used. Required by microsoft/vscode-v8-heap-tools - ios_printf(json, "\"trace_function_infos\":[],"); - ios_printf(json, "\"trace_tree\":[],"); - ios_printf(json, "\"samples\":[],"); - ios_printf(json, "\"locations\":[]"); - // end not used + ios_printf(json, " \"node_count\":%zu,\n", snapshot.num_nodes); + ios_printf(json, " \"edge_count\":%zu,\n", snapshot.num_edges); + ios_printf(json, " \"trace_function_count\":0\n"); // not used. Required by microsoft/vscode-v8-heap-tools + ios_printf(json, "}\n"); // end "snapshot" + // this } is removed by the julia reassembler in Profile ios_printf(json, "}"); } diff --git a/stdlib/Profile/src/heapsnapshot_reassemble.jl b/stdlib/Profile/src/heapsnapshot_reassemble.jl index 2413ae538b8ac..b2d86ee1f27b6 100644 --- a/stdlib/Profile/src/heapsnapshot_reassemble.jl +++ b/stdlib/Profile/src/heapsnapshot_reassemble.jl @@ -155,7 +155,8 @@ function assemble_snapshot(in_prefix, io::IO) _write_decimal_number(io, nodes.edge_count[i], _digits_buf) print(io, ",0,0") end - print(io, "],\"edges\":[") + print(io, "],\n") + print(io, "\"edges\":[") e = 1 for n in 1:length(nodes) count = nodes.edge_count[n] @@ -177,6 +178,13 @@ function assemble_snapshot(in_prefix, io::IO) end println(io, "],") + # not used. Required by microsoft/vscode-v8-heap-tools + # This order of these fields is required by chrome dev tools otherwise loading fails + println(io, "\"trace_function_infos\":[],") + println(io, "\"trace_tree\":[],") + println(io, "\"samples\":[],") + println(io, "\"locations\":[],") + println(io, "\"strings\":[") open(string(in_prefix, ".strings"), "r") do strings_io first = true From 4f262862893380ed3c7abed883172a129539dae1 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:34:58 +0900 Subject: [PATCH 06/15] inference: add missing `TypeVar` handling for `instanceof_tfunc` (#55884) I thought these sort of problems had been addressed by d60f92c, but it seems some were missed. Specifically, `t.a` and `t.b` from `t::Union` could be `TypeVar`, and if they are passed to a subroutine or recursed without being unwrapped or rewrapped, errors like JuliaLang/julia#55882 could occur. This commit resolves the issue by calling `unwraptv` in the `Union` handling within `instanceof_tfunc`. I also found a similar issue inside `nfields_tfunc`, so that has also been fixed, and test cases have been added. While I haven't been able to make up a test case specifically for the fix in `instanceof_tfunc`, I have confirmed that this commit certainly fixes the issue reported in JuliaLang/julia#55882. - fixes JuliaLang/julia#55882 (cherry picked from commit 4b27a169bda6ac970fc677962c30af51a6a9ca74) --- base/compiler/tfuncs.jl | 8 ++++---- test/compiler/inference.jl | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 1fded9d33b178..97d492304f678 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -134,8 +134,8 @@ function instanceof_tfunc(@nospecialize(t), astag::Bool=false, @nospecialize(tro end return tr, isexact, isconcrete, istype elseif isa(t, Union) - ta, isexact_a, isconcrete_a, istype_a = instanceof_tfunc(t.a, astag, troot) - tb, isexact_b, isconcrete_b, istype_b = instanceof_tfunc(t.b, astag, troot) + ta, isexact_a, isconcrete_a, istype_a = instanceof_tfunc(unwraptv(t.a), astag, troot) + tb, isexact_b, isconcrete_b, istype_b = instanceof_tfunc(unwraptv(t.b), astag, troot) isconcrete = isconcrete_a && isconcrete_b istype = istype_a && istype_b # most users already handle the Union case, so here we assume that @@ -549,9 +549,9 @@ add_tfunc(Core.sizeof, 1, 1, sizeof_tfunc, 1) end end if isa(x, Union) - na = nfields_tfunc(𝕃, x.a) + na = nfields_tfunc(𝕃, unwraptv(x.a)) na === Int && return Int - return tmerge(na, nfields_tfunc(𝕃, x.b)) + return tmerge(𝕃, na, nfields_tfunc(𝕃, unwraptv(x.b))) end return Int end diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index a849ddbf982a8..eadbe719cc961 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -5682,3 +5682,6 @@ end t155751 = Union{AbstractArray{UInt8, 4}, Array{Float32, 4}, Grid55751{Float32, 3, _A} where _A} t255751 = Array{Float32, 3} @test Core.Compiler.tmerge_types_slow(t155751,t255751) == AbstractArray # shouldn't hang + +issue55882_nfields(x::Union{T,Nothing}) where T<:Number = nfields(x) +@test Base.infer_return_type(issue55882_nfields) <: Int From 73d43afea4126c1cc397649c3d9902a643026cc6 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Fri, 27 Sep 2024 10:29:32 +0200 Subject: [PATCH 07/15] Install terminfo data under /usr/share/julia (#55881) Just like all other libraries, we don't want internal Julia files to mess with system files. Introduced by https://github.com/JuliaLang/julia/pull/55411. (cherry picked from commit 0dbb6eb679c1c124c212ae9ce399004873041cf1) --- Makefile | 2 +- base/terminfo.jl | 2 +- deps/terminfo.mk | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ec13f23bbb2cf..05683fda0a004 100644 --- a/Makefile +++ b/Makefile @@ -405,7 +405,7 @@ endif $(INSTALL_F) $(JULIAHOME)/contrib/julia.appdata.xml $(DESTDIR)$(datarootdir)/appdata/ # Install terminal info database ifneq ($(WITH_TERMINFO),0) - cp -R -L $(build_datarootdir)/terminfo $(DESTDIR)$(datarootdir) + cp -R -L $(build_datarootdir)/julia/terminfo $(DESTDIR)$(datarootdir)/julia/ endif # Update RPATH entries and JL_SYSTEM_IMAGE_PATH if $(private_libdir_rel) != $(build_private_libdir_rel) diff --git a/base/terminfo.jl b/base/terminfo.jl index 79713f4a86aa3..8ea8387077d36 100644 --- a/base/terminfo.jl +++ b/base/terminfo.jl @@ -262,7 +262,7 @@ function find_terminfo_file(term::String) append!(terminfo_dirs, replace(split(ENV["TERMINFO_DIRS"], ':'), "" => "/usr/share/terminfo")) - push!(terminfo_dirs, normpath(Sys.BINDIR, DATAROOTDIR, "terminfo")) + push!(terminfo_dirs, normpath(Sys.BINDIR, DATAROOTDIR, "julia", "terminfo")) Sys.isunix() && push!(terminfo_dirs, "/etc/terminfo", "/lib/terminfo", "/usr/share/terminfo") for dir in terminfo_dirs diff --git a/deps/terminfo.mk b/deps/terminfo.mk index 63194f786f566..60865838a813e 100644 --- a/deps/terminfo.mk +++ b/deps/terminfo.mk @@ -22,8 +22,8 @@ $(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)/build-checked: $(BUILDDIR)/TermInfoDB-v$ echo 1 > $@ define TERMINFO_INSTALL - mkdir -p $2/$$(build_datarootdir) - cp -R $1/terminfo $2/$$(build_datarootdir) + mkdir -p $2/$$(build_datarootdir)/julia + cp -R $1/terminfo $2/$$(build_datarootdir)/julia/ endef $(eval $(call staged-install, \ terminfo,TermInfoDB-v$(TERMINFO_VER), \ From b3181b8f0a71df57dac45756a03818170aeecd85 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 27 Sep 2024 21:49:29 -0400 Subject: [PATCH 08/15] do not intentionally suppress errors in precompile script from being reported or failing the result (#55909) I was slightly annoying that the build was set up to succeed if this step failed, so I removed the error suppression and fixed up the script slightly (cherry picked from commit 4a4ca9c815207a80ea81b884b196dfeafc3cb877) --- contrib/generate_precompile.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index e4eeeed577686..3aeb2d5b2c0ad 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -342,8 +342,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe print_state("step1" => "F$n_step1") return :ok end - Base.errormonitor(step1) - !PARALLEL_PRECOMPILATION && wait(step1) + PARALLEL_PRECOMPILATION ? bind(statements_step1, step1) : wait(step1) # Create a staging area where all the loaded packages are available PrecompileStagingArea = Module() @@ -357,7 +356,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe # Make statements unique statements = Set{String}() # Execute the precompile statements - for sts in [statements_step1,], statement in sts + for statement in statements_step1 # Main should be completely clean occursin("Main.", statement) && continue Base.in!(statement, statements) && continue @@ -393,6 +392,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe println() # Seems like a reasonable number right now, adjust as needed # comment out if debugging script + have_repl = false n_succeeded > (have_repl ? 650 : 90) || @warn "Only $n_succeeded precompile statements" fetch(step1) == :ok || throw("Step 1 of collecting precompiles failed.") @@ -403,7 +403,6 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe finally fancyprint && print(ansi_enablecursor) GC.gc(true); GC.gc(false); # reduce memory footprint - return end generate_precompile_statements() From a4c3a19b6e28006fcac77424e169d0bdca3c267e Mon Sep 17 00:00:00 2001 From: Neven Sajko Date: Sun, 29 Sep 2024 11:41:59 +0200 Subject: [PATCH 09/15] fix rawbigints OOB issues (#55917) Fixes issues introduced in #50691 and found in #55906: * use `@inbounds` and `@boundscheck` macros in rawbigints, for catching OOB with `--check-bounds=yes` * fix OOB in `truncate` (cherry picked from commit 17445fe752b7b99633ca306af0981baca9f66bda) --- base/rawbigints.jl | 31 ++++++++++++++++++++++--------- test/mpfr.jl | 9 +++++++++ 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/base/rawbigints.jl b/base/rawbigints.jl index 6508bea05be0f..a9bb18e163e2d 100644 --- a/base/rawbigints.jl +++ b/base/rawbigints.jl @@ -21,14 +21,21 @@ reversed_index(n::Int, i::Int) = n - i - 1 reversed_index(x, i::Int, v::Val) = reversed_index(elem_count(x, v), i)::Int split_bit_index(x::RawBigInt, i::Int) = divrem(i, word_length(x), RoundToZero) +function get_elem_words_raw(x::RawBigInt{T}, i::Int) where {T} + @boundscheck if (i < 0) || (elem_count(x, Val(:words)) ≤ i) + throw(BoundsError(x, i)) + end + d = x.d + j = i + 1 + (GC.@preserve d unsafe_load(Ptr{T}(pointer(d)), j))::T +end + """ `i` is the zero-based index of the wanted word in `x`, starting from the less significant words. """ -function get_elem(x::RawBigInt{T}, i::Int, ::Val{:words}, ::Val{:ascending}) where {T} - # `i` must be non-negative and less than `x.word_count` - d = x.d - (GC.@preserve d unsafe_load(Ptr{T}(pointer(d)), i + 1))::T +function get_elem(x::RawBigInt, i::Int, ::Val{:words}, ::Val{:ascending}) + @inbounds @inline get_elem_words_raw(x, i) end function get_elem(x, i::Int, v::Val, ::Val{:descending}) @@ -96,7 +103,8 @@ end """ Returns an integer of type `R`, consisting of the `len` most -significant bits of `x`. +significant bits of `x`. If there are less than `len` bits in `x`, +the least significant bits are zeroed. """ function truncated(::Type{R}, x::RawBigInt, len::Int) where {R<:Integer} ret = zero(R) @@ -104,17 +112,22 @@ function truncated(::Type{R}, x::RawBigInt, len::Int) where {R<:Integer} word_count, bit_count_in_word = split_bit_index(x, len) k = word_length(x) vals = (Val(:words), Val(:descending)) + lenx = elem_count(x, first(vals)) for w ∈ 0:(word_count - 1) ret <<= k - word = get_elem(x, w, vals...) - ret |= R(word) + if w < lenx + word = get_elem(x, w, vals...) + ret |= R(word) + end end if !iszero(bit_count_in_word) ret <<= bit_count_in_word - wrd = get_elem(x, word_count, vals...) - ret |= R(wrd >>> (k - bit_count_in_word)) + if word_count < lenx + wrd = get_elem(x, word_count, vals...) + ret |= R(wrd >>> (k - bit_count_in_word)) + end end end ret::R diff --git a/test/mpfr.jl b/test/mpfr.jl index a0dd15d97f70c..63365cf803deb 100644 --- a/test/mpfr.jl +++ b/test/mpfr.jl @@ -1046,3 +1046,12 @@ end @test Float16(bf) == Float16(2.0e-7) end end + +@testset "RawBigInt truncation OOB read" begin + @testset "T: $T" for T ∈ (UInt8, UInt16, UInt32, UInt64, UInt128) + v = Base.RawBigInt{T}("a"^sizeof(T), 1) + @testset "bit_count: $bit_count" for bit_count ∈ (0:10:80) + @test Base.truncated(UInt128, v, bit_count) isa Any + end + end +end From 2b28354b6c8ac1295b4a251fdf8cbc4bd36ecba1 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Mon, 30 Sep 2024 12:38:20 +0200 Subject: [PATCH 10/15] bump Pkg to latest 1.11 --- .../Pkg-69938284d6804dd30a24577397f52dfdedbd37e1.tar.gz/md5 | 1 - .../Pkg-69938284d6804dd30a24577397f52dfdedbd37e1.tar.gz/sha512 | 1 - .../Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/md5 | 1 + .../Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-69938284d6804dd30a24577397f52dfdedbd37e1.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-69938284d6804dd30a24577397f52dfdedbd37e1.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/md5 create mode 100644 deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/sha512 diff --git a/deps/checksums/Pkg-69938284d6804dd30a24577397f52dfdedbd37e1.tar.gz/md5 b/deps/checksums/Pkg-69938284d6804dd30a24577397f52dfdedbd37e1.tar.gz/md5 deleted file mode 100644 index 12ef870b31933..0000000000000 --- a/deps/checksums/Pkg-69938284d6804dd30a24577397f52dfdedbd37e1.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -248ce4c4b85394431e71b6f555f82216 diff --git a/deps/checksums/Pkg-69938284d6804dd30a24577397f52dfdedbd37e1.tar.gz/sha512 b/deps/checksums/Pkg-69938284d6804dd30a24577397f52dfdedbd37e1.tar.gz/sha512 deleted file mode 100644 index b51345f090447..0000000000000 --- a/deps/checksums/Pkg-69938284d6804dd30a24577397f52dfdedbd37e1.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -84f193d72062199f2791ba7f7c2b6eb547e1a97569ae20cbb3557bc291715dd0cf55b2e121c4829b4572929e97255fd53b9210a168a7fde36f09ae6cf76c7f40 diff --git a/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/md5 b/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/md5 new file mode 100644 index 0000000000000..329bb174c9e2e --- /dev/null +++ b/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/md5 @@ -0,0 +1 @@ +84d6a319d77087c3e6885b547e562dcd diff --git a/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/sha512 b/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/sha512 new file mode 100644 index 0000000000000..323183dd21083 --- /dev/null +++ b/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/sha512 @@ -0,0 +1 @@ +490b3797e1cb49a1335a732b9c9b5274fec00bf3a28153af0979f3238d79cbaa9155f2f05004b0c81f3116f1434341e3013e2ae0ccf7ef1f73bd08f2909d72ad diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 5729fc9f7c69d..fea88d7cc679c 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = release-1.11 -PKG_SHA1 = 69938284d6804dd30a24577397f52dfdedbd37e1 +PKG_SHA1 = a1904c128feb19e4d3e24bac2fd5ca6bf4721539 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 108bd2e9f95bb1f649a05b05f435a14156eacdb4 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Mon, 30 Sep 2024 18:31:04 +0530 Subject: [PATCH 11/15] Broadcast binary ops involving strided triangular #55798 --- stdlib/LinearAlgebra/src/symmetric.jl | 8 +-- stdlib/LinearAlgebra/src/triangular.jl | 81 +++++++++++++++++++++----- stdlib/LinearAlgebra/test/symmetric.jl | 25 ++++++++ 3 files changed, 94 insertions(+), 20 deletions(-) diff --git a/stdlib/LinearAlgebra/src/symmetric.jl b/stdlib/LinearAlgebra/src/symmetric.jl index 8cc07566c109e..d6b2e860d1a06 100644 --- a/stdlib/LinearAlgebra/src/symmetric.jl +++ b/stdlib/LinearAlgebra/src/symmetric.jl @@ -536,10 +536,10 @@ for f in (:+, :-) @eval begin $f(A::Hermitian, B::Symmetric{<:Real}) = $f(A, Hermitian(parent(B), sym_uplo(B.uplo))) $f(A::Symmetric{<:Real}, B::Hermitian) = $f(Hermitian(parent(A), sym_uplo(A.uplo)), B) - $f(A::SymTridiagonal, B::Symmetric) = Symmetric($f(A, B.data), sym_uplo(B.uplo)) - $f(A::Symmetric, B::SymTridiagonal) = Symmetric($f(A.data, B), sym_uplo(A.uplo)) - $f(A::SymTridiagonal{<:Real}, B::Hermitian) = Hermitian($f(A, B.data), sym_uplo(B.uplo)) - $f(A::Hermitian, B::SymTridiagonal{<:Real}) = Hermitian($f(A.data, B), sym_uplo(A.uplo)) + $f(A::SymTridiagonal, B::Symmetric) = $f(Symmetric(A, sym_uplo(B.uplo)), B) + $f(A::Symmetric, B::SymTridiagonal) = $f(A, Symmetric(B, sym_uplo(A.uplo))) + $f(A::SymTridiagonal{<:Real}, B::Hermitian) = $f(Hermitian(A, sym_uplo(B.uplo)), B) + $f(A::Hermitian, B::SymTridiagonal{<:Real}) = $f(A, Hermitian(B, sym_uplo(A.uplo))) end end diff --git a/stdlib/LinearAlgebra/src/triangular.jl b/stdlib/LinearAlgebra/src/triangular.jl index fb9f6b2df9121..b1de141b109c0 100644 --- a/stdlib/LinearAlgebra/src/triangular.jl +++ b/stdlib/LinearAlgebra/src/triangular.jl @@ -779,24 +779,73 @@ fillstored!(A::UpperTriangular, x) = (fillband!(A.data, x, 0, size(A,2)-1); fillstored!(A::UnitUpperTriangular, x) = (fillband!(A.data, x, 1, size(A,2)-1); A) # Binary operations -+(A::UpperTriangular, B::UpperTriangular) = UpperTriangular(A.data + B.data) -+(A::LowerTriangular, B::LowerTriangular) = LowerTriangular(A.data + B.data) -+(A::UpperTriangular, B::UnitUpperTriangular) = UpperTriangular(A.data + triu(B.data, 1) + I) -+(A::LowerTriangular, B::UnitLowerTriangular) = LowerTriangular(A.data + tril(B.data, -1) + I) -+(A::UnitUpperTriangular, B::UpperTriangular) = UpperTriangular(triu(A.data, 1) + B.data + I) -+(A::UnitLowerTriangular, B::LowerTriangular) = LowerTriangular(tril(A.data, -1) + B.data + I) -+(A::UnitUpperTriangular, B::UnitUpperTriangular) = UpperTriangular(triu(A.data, 1) + triu(B.data, 1) + 2I) -+(A::UnitLowerTriangular, B::UnitLowerTriangular) = LowerTriangular(tril(A.data, -1) + tril(B.data, -1) + 2I) +# use broadcasting if the parents are strided, where we loop only over the triangular part +function +(A::UpperTriangular, B::UpperTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .+ B + UpperTriangular(A.data + B.data) +end +function +(A::LowerTriangular, B::LowerTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .+ B + LowerTriangular(A.data + B.data) +end +function +(A::UpperTriangular, B::UnitUpperTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .+ B + UpperTriangular(A.data + triu(B.data, 1) + I) +end +function +(A::LowerTriangular, B::UnitLowerTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .+ B + LowerTriangular(A.data + tril(B.data, -1) + I) +end +function +(A::UnitUpperTriangular, B::UpperTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .+ B + UpperTriangular(triu(A.data, 1) + B.data + I) +end +function +(A::UnitLowerTriangular, B::LowerTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .+ B + LowerTriangular(tril(A.data, -1) + B.data + I) +end +function +(A::UnitUpperTriangular, B::UnitUpperTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .+ B + UpperTriangular(triu(A.data, 1) + triu(B.data, 1) + 2I) +end +function +(A::UnitLowerTriangular, B::UnitLowerTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .+ B + LowerTriangular(tril(A.data, -1) + tril(B.data, -1) + 2I) +end +(A::AbstractTriangular, B::AbstractTriangular) = copyto!(similar(parent(A)), A) + copyto!(similar(parent(B)), B) --(A::UpperTriangular, B::UpperTriangular) = UpperTriangular(A.data - B.data) --(A::LowerTriangular, B::LowerTriangular) = LowerTriangular(A.data - B.data) --(A::UpperTriangular, B::UnitUpperTriangular) = UpperTriangular(A.data - triu(B.data, 1) - I) --(A::LowerTriangular, B::UnitLowerTriangular) = LowerTriangular(A.data - tril(B.data, -1) - I) --(A::UnitUpperTriangular, B::UpperTriangular) = UpperTriangular(triu(A.data, 1) - B.data + I) --(A::UnitLowerTriangular, B::LowerTriangular) = LowerTriangular(tril(A.data, -1) - B.data + I) --(A::UnitUpperTriangular, B::UnitUpperTriangular) = UpperTriangular(triu(A.data, 1) - triu(B.data, 1)) --(A::UnitLowerTriangular, B::UnitLowerTriangular) = LowerTriangular(tril(A.data, -1) - tril(B.data, -1)) +function -(A::UpperTriangular, B::UpperTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .- B + UpperTriangular(A.data - B.data) +end +function -(A::LowerTriangular, B::LowerTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .- B + LowerTriangular(A.data - B.data) +end +function -(A::UpperTriangular, B::UnitUpperTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .- B + UpperTriangular(A.data - triu(B.data, 1) - I) +end +function -(A::LowerTriangular, B::UnitLowerTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .- B + LowerTriangular(A.data - tril(B.data, -1) - I) +end +function -(A::UnitUpperTriangular, B::UpperTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .- B + UpperTriangular(triu(A.data, 1) - B.data + I) +end +function -(A::UnitLowerTriangular, B::LowerTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .- B + LowerTriangular(tril(A.data, -1) - B.data + I) +end +function -(A::UnitUpperTriangular, B::UnitUpperTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .- B + UpperTriangular(triu(A.data, 1) - triu(B.data, 1)) +end +function -(A::UnitLowerTriangular, B::UnitLowerTriangular) + (parent(A) isa StridedMatrix || parent(B) isa StridedMatrix) && return A .- B + LowerTriangular(tril(A.data, -1) - tril(B.data, -1)) +end -(A::AbstractTriangular, B::AbstractTriangular) = copyto!(similar(parent(A)), A) - copyto!(similar(parent(B)), B) # use broadcasting if the parents are strided, where we loop only over the triangular part diff --git a/stdlib/LinearAlgebra/test/symmetric.jl b/stdlib/LinearAlgebra/test/symmetric.jl index 25522042dcf2b..acf202fba0937 100644 --- a/stdlib/LinearAlgebra/test/symmetric.jl +++ b/stdlib/LinearAlgebra/test/symmetric.jl @@ -1014,4 +1014,29 @@ end end end +@testset "partly iniitalized matrices" begin + a = Matrix{BigFloat}(undef, 2,2) + a[1] = 1; a[3] = 1; a[4] = 1 + h = Hermitian(a) + s = Symmetric(a) + d = Diagonal([1,1]) + symT = SymTridiagonal([1 1;1 1]) + @test h+d == Array(h) + Array(d) + @test h+symT == Array(h) + Array(symT) + @test s+d == Array(s) + Array(d) + @test s+symT == Array(s) + Array(symT) + @test h-d == Array(h) - Array(d) + @test h-symT == Array(h) - Array(symT) + @test s-d == Array(s) - Array(d) + @test s-symT == Array(s) - Array(symT) + @test d+h == Array(d) + Array(h) + @test symT+h == Array(symT) + Array(h) + @test d+s == Array(d) + Array(s) + @test symT+s == Array(symT) + Array(s) + @test d-h == Array(d) - Array(h) + @test symT-h == Array(symT) - Array(h) + @test d-s == Array(d) - Array(s) + @test symT-s == Array(symT) - Array(s) +end + end # module TestSymmetric From d5e8e1f25650cfe0dc9212f427d079938a36b79a Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:02:59 -0400 Subject: [PATCH 12/15] TOML: Avoid type-pirating `Base.TOML.Parser` (#55892) Since stdlibs can be duplicated but Base never is, `Base.require_stdlib` makes type piracy even more complicated than it normally would be. To adapt, this changes `TOML.Parser` to be a type defined by the TOML stdlib, so that we can define methods on it without committing type-piracy and avoid problems like Pkg.jl#4017 Resolves https://github.com/JuliaLang/Pkg.jl/issues/4017#issuecomment-2377589989 (cherry picked from commit 2a2878c143b87e5184565c895d090aab6e9017e9) --- stdlib/TOML/src/TOML.jl | 43 ++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/stdlib/TOML/src/TOML.jl b/stdlib/TOML/src/TOML.jl index 94d2808c0bc24..b37a5ca83c251 100644 --- a/stdlib/TOML/src/TOML.jl +++ b/stdlib/TOML/src/TOML.jl @@ -25,7 +25,7 @@ module Internals end # https://github.com/JuliaLang/julia/issues/36605 -readstring(f::AbstractString) = isfile(f) ? read(f, String) : error(repr(f), ": No such file") +_readstring(f::AbstractString) = isfile(f) ? read(f, String) : error(repr(f), ": No such file") """ Parser() @@ -36,12 +36,14 @@ explicitly create a `Parser` but instead one directly use use will however reuse some internal data structures which can be beneficial for performance if a larger number of small files are parsed. """ -const Parser = Internals.Parser +struct Parser + _p::Internals.Parser{Dates} +end # Dates-enabled constructors -Parser() = Parser{Dates}() -Parser(io::IO) = Parser{Dates}(io) -Parser(str::String; filepath=nothing) = Parser{Dates}(str; filepath) +Parser() = Parser(Internals.Parser{Dates}()) +Parser(io::IO) = Parser(Internals.Parser{Dates}(io)) +Parser(str::String; filepath=nothing) = Parser(Internals.Parser{Dates}(str; filepath)) """ parsefile(f::AbstractString) @@ -53,9 +55,9 @@ Parse file `f` and return the resulting table (dictionary). Throw a See also [`TOML.tryparsefile`](@ref). """ parsefile(f::AbstractString) = - Internals.parse(Parser(readstring(f); filepath=abspath(f))) + Internals.parse(Internals.Parser{Dates}(_readstring(f); filepath=abspath(f))) parsefile(p::Parser, f::AbstractString) = - Internals.parse(Internals.reinit!(p, readstring(f); filepath=abspath(f))) + Internals.parse(Internals.reinit!(p._p, _readstring(f); filepath=abspath(f))) """ tryparsefile(f::AbstractString) @@ -67,9 +69,9 @@ Parse file `f` and return the resulting table (dictionary). Return a See also [`TOML.parsefile`](@ref). """ tryparsefile(f::AbstractString) = - Internals.tryparse(Parser(readstring(f); filepath=abspath(f))) + Internals.tryparse(Internals.Parser{Dates}(_readstring(f); filepath=abspath(f))) tryparsefile(p::Parser, f::AbstractString) = - Internals.tryparse(Internals.reinit!(p, readstring(f); filepath=abspath(f))) + Internals.tryparse(Internals.reinit!(p._p, _readstring(f); filepath=abspath(f))) """ parse(x::Union{AbstractString, IO}) @@ -80,10 +82,11 @@ Throw a [`ParserError`](@ref) upon failure. See also [`TOML.tryparse`](@ref). """ +parse(p::Parser) = Internals.parse(p._p) parse(str::AbstractString) = - Internals.parse(Parser(String(str))) + Internals.parse(Internals.Parser{Dates}(String(str))) parse(p::Parser, str::AbstractString) = - Internals.parse(Internals.reinit!(p, String(str))) + Internals.parse(Internals.reinit!(p._p, String(str))) parse(io::IO) = parse(read(io, String)) parse(p::Parser, io::IO) = parse(p, read(io, String)) @@ -96,10 +99,11 @@ Return a [`ParserError`](@ref) upon failure. See also [`TOML.parse`](@ref). """ +tryparse(p::Parser) = Internals.tryparse(p._p) tryparse(str::AbstractString) = - Internals.tryparse(Parser(String(str))) + Internals.tryparse(Internals.Parser{Dates}(String(str))) tryparse(p::Parser, str::AbstractString) = - Internals.tryparse(Internals.reinit!(p, String(str))) + Internals.tryparse(Internals.reinit!(p._p, String(str))) tryparse(io::IO) = tryparse(read(io, String)) tryparse(p::Parser, io::IO) = tryparse(p, read(io, String)) @@ -131,4 +135,17 @@ supported type. """ const print = Internals.Printer.print +public Parser, parsefile, tryparsefile, parse, tryparse, ParserError, print + +# These methods are private Base interfaces, but we do our best to support them over +# the TOML stdlib types anyway to minimize downstream breakage. +Base.TOMLCache(p::Parser) = Base.TOMLCache(p._p, Dict{String, Base.CachedTOMLDict}()) +Base.TOMLCache(p::Parser, d::Base.CachedTOMLDict) = Base.TOMLCache(p._p, d) +Base.TOMLCache(p::Parser, d::Dict{String, Dict{String, Any}}) = Base.TOMLCache(p._p, d) + +Internals.reinit!(p::Parser, str::String; filepath::Union{Nothing, String}=nothing) = + Internals.reinit!(p._p, str; filepath) +Internals.parse(p::Parser) = Internals.parse(p._p) +Internals.tryparse(p::Parser) = Internals.tryparse(p._p) + end From 23b7de613796ca81465043335eb001a7d983986d Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 1 Oct 2024 01:41:23 +0200 Subject: [PATCH 13/15] relocation: account for trailing path separator in depot paths (#55355) Fixes #55340 (cherry picked from commit a7c5056b722182adfd183fdc7bdfdef39cd8e28e) --- base/loading.jl | 26 ++++++++++++++++---------- src/precompile.c | 17 +++++++++++++---- src/staticdata_utils.c | 20 ++++++++++++++------ test/relocatedepot.jl | 34 +++++++++++++++++++++++++++++----- 4 files changed, 72 insertions(+), 25 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 0262ace997a09..c5db12d76fc26 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -3119,16 +3119,9 @@ mutable struct CacheHeaderIncludes const modpath::Vector{String} # seemingly not needed in Base, but used by Revise end -function replace_depot_path(path::AbstractString) - for depot in DEPOT_PATH - !isdir(depot) && continue - - # Strip extraneous pathseps through normalization. - if isdirpath(depot) - depot = dirname(depot) - end - - if startswith(path, depot) +function replace_depot_path(path::AbstractString, depots::Vector{String}=normalize_depots_for_relocation()) + for depot in depots + if startswith(path, string(depot, Filesystem.pathsep())) || path == depot path = replace(path, depot => "@depot"; count=1) break end @@ -3136,6 +3129,19 @@ function replace_depot_path(path::AbstractString) return path end +function normalize_depots_for_relocation() + depots = String[] + sizehint!(depots, length(DEPOT_PATH)) + for d in DEPOT_PATH + isdir(d) || continue + if isdirpath(d) + d = dirname(d) + end + push!(depots, abspath(d)) + end + return depots +end + function restore_depot_path(path::AbstractString, depot::AbstractString) replace(path, r"^@depot" => depot; count=1) end diff --git a/src/precompile.c b/src/precompile.c index c40e867ea699e..ec4f7a69002c1 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -39,9 +39,17 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { static jl_value_t *replace_depot_func = NULL; if (!replace_depot_func) replace_depot_func = jl_get_global(jl_base_module, jl_symbol("replace_depot_path")); + static jl_value_t *normalize_depots_func = NULL; + if (!normalize_depots_func) + normalize_depots_func = jl_get_global(jl_base_module, jl_symbol("normalize_depots_for_relocation")); ios_t srctext; - jl_value_t *deptuple = NULL; - JL_GC_PUSH2(&deptuple, &udeps); + jl_value_t *deptuple = NULL, *depots = NULL; + JL_GC_PUSH3(&deptuple, &udeps, &depots); + jl_task_t *ct = jl_current_task; + size_t last_age = ct->world_age; + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + depots = jl_apply(&normalize_depots_func, 1); + ct->world_age = last_age; for (size_t i = 0; i < len; i++) { deptuple = jl_array_ptr_ref(udeps, i); jl_value_t *depmod = jl_fieldref(deptuple, 0); // module @@ -60,13 +68,14 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { } jl_value_t **replace_depot_args; - JL_GC_PUSHARGS(replace_depot_args, 2); + JL_GC_PUSHARGS(replace_depot_args, 3); replace_depot_args[0] = replace_depot_func; replace_depot_args[1] = abspath; + replace_depot_args[2] = depots; jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - jl_value_t *depalias = (jl_value_t*)jl_apply(replace_depot_args, 2); + jl_value_t *depalias = (jl_value_t*)jl_apply(replace_depot_args, 3); ct->world_age = last_age; JL_GC_POP(); diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 8d3d071054083..66ef05963bbe3 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -752,6 +752,16 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t static jl_value_t *replace_depot_func = NULL; if (!replace_depot_func) replace_depot_func = jl_get_global(jl_base_module, jl_symbol("replace_depot_path")); + static jl_value_t *normalize_depots_func = NULL; + if (!normalize_depots_func) + normalize_depots_func = jl_get_global(jl_base_module, jl_symbol("normalize_depots_for_relocation")); + + jl_value_t *depots = NULL, *prefs_hash = NULL, *prefs_list = NULL; + JL_GC_PUSH2(&depots, &prefs_list); + last_age = ct->world_age; + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + depots = jl_apply(&normalize_depots_func, 1); + ct->world_age = last_age; // write a placeholder for total size so that we can quickly seek past all of the // dependencies if we don't need them @@ -764,13 +774,14 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t if (replace_depot_func) { jl_value_t **replace_depot_args; - JL_GC_PUSHARGS(replace_depot_args, 2); + JL_GC_PUSHARGS(replace_depot_args, 3); replace_depot_args[0] = replace_depot_func; replace_depot_args[1] = deppath; + replace_depot_args[2] = depots; ct = jl_current_task; size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - deppath = (jl_value_t*)jl_apply(replace_depot_args, 2); + deppath = (jl_value_t*)jl_apply(replace_depot_args, 3); ct->world_age = last_age; JL_GC_POP(); } @@ -803,9 +814,6 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t write_int32(s, 0); // terminator, for ease of reading // Calculate Preferences hash for current package. - jl_value_t *prefs_hash = NULL; - jl_value_t *prefs_list = NULL; - JL_GC_PUSH1(&prefs_list); if (jl_base_module) { // Toplevel module is the module we're currently compiling, use it to get our preferences hash jl_value_t * toplevel = (jl_value_t*)jl_get_global(jl_base_module, jl_symbol("__toplevel__")); @@ -852,7 +860,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t write_int32(s, 0); write_uint64(s, 0); } - JL_GC_POP(); // for prefs_list + JL_GC_POP(); // for depots, prefs_list // write a dummy file position to indicate the beginning of the source-text pos = ios_pos(s); diff --git a/test/relocatedepot.jl b/test/relocatedepot.jl index 039d422c35e25..2ef6dec90dbc1 100644 --- a/test/relocatedepot.jl +++ b/test/relocatedepot.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + using Test @@ -26,16 +28,38 @@ end if !test_relocated_depot - @testset "insert @depot tag in path" begin + @testset "edge cases when inserting @depot tag in path" begin + # insert @depot only once for first match test_harness() do mktempdir() do dir pushfirst!(DEPOT_PATH, dir) - path = dir*dir - @test Base.replace_depot_path(path) == "@depot"*dir + if Sys.iswindows() + # dirs start with a drive letter instead of a path separator + path = dir*Base.Filesystem.pathsep()*dir + @test Base.replace_depot_path(path) == "@depot"*Base.Filesystem.pathsep()*dir + else + path = dir*dir + @test Base.replace_depot_path(path) == "@depot"*dir + end + end + + # 55340 + empty!(DEPOT_PATH) + mktempdir() do dir + jlrc = joinpath(dir, "julia-rc2") + jl = joinpath(dir, "julia") + mkdir(jl) + push!(DEPOT_PATH, jl) + @test Base.replace_depot_path(jl) == "@depot" + @test Base.replace_depot_path(string(jl,Base.Filesystem.pathsep())) == + string("@depot",Base.Filesystem.pathsep()) + @test Base.replace_depot_path(jlrc) != "@depot-rc2" + @test Base.replace_depot_path(jlrc) == jlrc end end + # deal with and without trailing path separators test_harness() do mktempdir() do dir pushfirst!(DEPOT_PATH, dir) @@ -43,9 +67,9 @@ if !test_relocated_depot if isdirpath(DEPOT_PATH[1]) DEPOT_PATH[1] = dirname(DEPOT_PATH[1]) # strip trailing pathsep end - tag = joinpath("@depot", "") # append a pathsep + tag = string("@depot", Base.Filesystem.pathsep()) @test startswith(Base.replace_depot_path(path), tag) - DEPOT_PATH[1] = joinpath(DEPOT_PATH[1], "") # append a pathsep + DEPOT_PATH[1] = string(DEPOT_PATH[1], Base.Filesystem.pathsep()) @test startswith(Base.replace_depot_path(path), tag) popfirst!(DEPOT_PATH) @test !startswith(Base.replace_depot_path(path), tag) From 9cb1ea6e15ae79003bb588099d5a66f11d36438a Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Tue, 1 Oct 2024 10:13:29 +0530 Subject: [PATCH 14/15] Limit `@inbounds` to indexing in the dual-iterator branch in `copyto_unaliased!` (#55919) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This simplifies the `copyto_unalised!` implementation where the source and destination have different `IndexStyle`s, and limits the `@inbounds` to only the indexing operation. In particular, the iteration over `eachindex(dest)` is not marked as `@inbounds` anymore. This seems to help with performance when the destination uses Cartesian indexing. Reduced implementation of the branch: ```julia function copyto_proposed!(dest, src) axes(dest) == axes(src) || throw(ArgumentError("incompatible sizes")) iterdest, itersrc = eachindex(dest), eachindex(src) for (destind, srcind) in zip(iterdest, itersrc) @inbounds dest[destind] = src[srcind] end dest end function copyto_current!(dest, src) axes(dest) == axes(src) || throw(ArgumentError("incompatible sizes")) iterdest, itersrc = eachindex(dest), eachindex(src) ret = iterate(iterdest) @inbounds for a in src idx, state = ret::NTuple{2,Any} dest[idx] = a ret = iterate(iterdest, state) end dest end function copyto_current_limitinbounds!(dest, src) axes(dest) == axes(src) || throw(ArgumentError("incompatible sizes")) iterdest, itersrc = eachindex(dest), eachindex(src) ret = iterate(iterdest) for isrc in itersrc idx, state = ret::NTuple{2,Any} @inbounds dest[idx] = src[isrc] ret = iterate(iterdest, state) end dest end ``` ```julia julia> a = zeros(40000,4000); b = rand(size(a)...); julia> av = view(a, UnitRange.(axes(a))...); julia> @btime copyto_current!($av, $b); 617.704 ms (0 allocations: 0 bytes) julia> @btime copyto_current_limitinbounds!($av, $b); 304.146 ms (0 allocations: 0 bytes) julia> @btime copyto_proposed!($av, $b); 240.217 ms (0 allocations: 0 bytes) julia> versioninfo() Julia Version 1.12.0-DEV.1260 Commit 4a4ca9c8152 (2024-09-28 01:49 UTC) Build Info: Official https://julialang.org release Platform Info: OS: Linux (x86_64-linux-gnu) CPU: 8 × Intel(R) Core(TM) i5-10310U CPU @ 1.70GHz WORD_SIZE: 64 LLVM: libLLVM-18.1.7 (ORCJIT, skylake) Threads: 1 default, 0 interactive, 1 GC (on 8 virtual cores) Environment: JULIA_EDITOR = subl ``` I'm not quite certain why the proposed implementation here (`copyto_proposed!`) is even faster than `copyto_current_limitinbounds!`. In any case, `copyto_proposed!` is easier to read, so I'm not complaining. This fixes https://github.com/JuliaLang/julia/issues/53158 (cherry picked from commit 06e7b9d292ed4ced5b523fe94daef30332eabbd3) --- base/abstractarray.jl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index ad70b973ab10b..bb4aff0f6a411 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1096,11 +1096,8 @@ function copyto_unaliased!(deststyle::IndexStyle, dest::AbstractArray, srcstyle: end else # Dual-iterator implementation - ret = iterate(iterdest) - @inbounds for a in src - idx, state = ret::NTuple{2,Any} - dest[idx] = a - ret = iterate(iterdest, state) + for (Idest, Isrc) in zip(iterdest, itersrc) + @inbounds dest[Idest] = src[Isrc] end end end From cb5cd2a7ad04317926faac441305261c55a094b5 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Tue, 1 Oct 2024 09:02:05 +0200 Subject: [PATCH 15/15] bump Pkg to latest v1.11 --- .../Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/md5 | 1 + .../Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/sha512 | 1 + .../Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/md5 | 1 - .../Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/sha512 | 1 - stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/md5 create mode 100644 deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/sha512 diff --git a/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/md5 b/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/md5 new file mode 100644 index 0000000000000..96d256e63fc13 --- /dev/null +++ b/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/md5 @@ -0,0 +1 @@ +83f567b5c65ae36ebd9b5878cf009513 diff --git a/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/sha512 b/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/sha512 new file mode 100644 index 0000000000000..c4b322eac0395 --- /dev/null +++ b/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/sha512 @@ -0,0 +1 @@ +bc1e6de2c679b862caeb5c1fb6c3187c9da4dc2e37dc89b5cdf1498c5a26fbe7516773b53c039927abd89a1a9e8215f410b08a3da9d1b6489fe9ca2da60e57d2 diff --git a/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/md5 b/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/md5 deleted file mode 100644 index 329bb174c9e2e..0000000000000 --- a/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -84d6a319d77087c3e6885b547e562dcd diff --git a/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/sha512 b/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/sha512 deleted file mode 100644 index 323183dd21083..0000000000000 --- a/deps/checksums/Pkg-a1904c128feb19e4d3e24bac2fd5ca6bf4721539.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -490b3797e1cb49a1335a732b9c9b5274fec00bf3a28153af0979f3238d79cbaa9155f2f05004b0c81f3116f1434341e3013e2ae0ccf7ef1f73bd08f2909d72ad diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index fea88d7cc679c..90a7c19aae59c 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = release-1.11 -PKG_SHA1 = a1904c128feb19e4d3e24bac2fd5ca6bf4721539 +PKG_SHA1 = 6ceafca8e2adab1b248b19e468d432c167dd7b7e PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1